Repository: MannLabs/alphaquant
Pull request #88: Fix issues with MaxQuant and Fragpipe tables
Author: login: ammarcsj
Labels: name: code-review
Created: 2025-03-14T11:54:42Z | Merged: 2025-03-17T13:13:28Z

Addressing issue #85 
- Changed the intable config for the GUI back to the default version also used in the python version
- Update config for MaxQuant evidence.txt
- Extend sample name extraction in GUI to parquet files
- Extend sample name extraction in GUI to wideformat tables (MaxQuant peptides.txt and fragpipe)
- Disable ML for wideformat tables
- Add quicktests that run short AlphaQuant runs on different MQ, Fragpipe, DIA-NN, Spectronaut


Diff:
diff --git a/alphaquant/config/quant_reader_config.yaml b/alphaquant/config/quant_reader_config.yaml
index c9841558..e47e8f24 100644
--- a/alphaquant/config/quant_reader_config.yaml
+++ b/alphaquant/config/quant_reader_config.yaml
@@ -144,36 +144,63 @@ maxquant_peptides_leading_razor_protein:
 
 maxquant_evidence:
   format: longtable
-  sample_ID: Experiment #Raw file
-  quant_ID: Intensity
+  sample_ID: Experiment
+  quant_ID:
+    precursor: Intensity
   protein_cols:
    - Gene names
-  ion_cols:
-   - Modified sequence
-   - Charge
+  ion_hierarchy:
+    precursor:
+      order: [SEQ, MOD, CHARGE]
+      mapping:
+        SEQ:
+          - Sequence
+        MOD:
+          - Mass
+        CHARGE:
+          - Charge
+  filters:
+    reverse:
+      param: Reverse
+      comparator: "!="
+      value: "+"
+    contaminant:
+      param: Potential contaminant
+      comparator: "!="
+      value: "+"
+  ml_level: SEQ
+  use_iontree: False
+
 
 maxquant_evidence_protein:
   format: longtable
-  sample_ID: Experiment #Raw file
-  quant_ID: Intensity
+  sample_ID: Experiment
+  quant_ID:
+    precursor: Intensity
   protein_cols:
    - Protein group IDs
-  ion_cols:
-   - Modified sequence
-   - Charge
-
-
+  ion_hierarchy:
+    precursor:
+      order: [SEQ, MOD, CHARGE]
+      mapping:
+        SEQ:
+          - Sequence
+        MOD:
+          - Mass
+        CHARGE:
+          - Charge
+  filters:
+    reverse:
+      param: Reverse
+      comparator: "!="
+      value: "+"
+    contaminant:
+      param: Potential contaminant
+      comparator: "!="
+      value: "+"
+  ml_level: SEQ
+  use_iontree: False
 
-maxquant_evidence_proteins_column:
-  format: longtable
-  sample_ID: Experiment #Raw file
-  quant_ID: Intensity
-  protein_cols:
-   - Proteins
-  ion_cols:
-   - Sequence
-   - Modifications
-   - Charge
 
 diann_precursor_fragion_ms1:
   format: longtable
@@ -1261,27 +1288,6 @@ diaumpire_precursor_ms1:
    - Peptide Key
 
 
-diann_wideformat:
-  format: widetable
-  protein_cols:
-   - Protein.Group
-  ion_cols:
-   - Stripped.Sequence
-   - Modified.Sequence
-   - Precursor.Charge
-  ion_hierarchy:
-    sequence_int:
-      order: [SEQ, MOD]
-      mapping:
-        SEQ:
-          - Stripped.Sequence
-        MOD:
-          - Modified.Sequence
-        CH:
-          - Precursor.Charge
-  ml_level: SEQ
-  use_iontree: False
-
 fragpipe_precursors:
   format: widetable
   quant_pre_or_suffix: " Intensity"
diff --git a/alphaquant/diffquant/condpair_analysis.py b/alphaquant/diffquant/condpair_analysis.py
index b86e82d2..3e81580b 100644
--- a/alphaquant/diffquant/condpair_analysis.py
+++ b/alphaquant/diffquant/condpair_analysis.py
@@ -49,7 +49,7 @@ def analyze_condpair(*,runconfig, condpair):
         write_out_normed_df(df_c1_normed, df_c2_normed, pep2prot, runconfig.results_dir, condpair)
     normed_c1 = aqbg.ConditionBackgrounds(df_c1_normed, p2z)
     normed_c2 = aqbg.ConditionBackgrounds(df_c2_normed, p2z)
-    
+
     ions_to_check = normed_c1.ion2nonNanvals.keys() & normed_c2.ion2nonNanvals.keys()
     ions_to_check = sorted(ions_to_check)
 
@@ -82,16 +82,16 @@ def analyze_condpair(*,runconfig, condpair):
         ions = prot2diffions.get(prot)
         if len(ions)<runconfig.min_num_ions:
             continue
-        
-        clustered_prot_node = aqclust.get_scored_clusterselected_ions(prot, ions, normed_c1, normed_c2, bgpair2diffDist, p2z, deedpair2doublediffdist, 
-                                                                        pval_threshold_basis = runconfig.cluster_threshold_pval, fcfc_threshold = runconfig.cluster_threshold_fcfc, 
+
+        clustered_prot_node = aqclust.get_scored_clusterselected_ions(prot, ions, normed_c1, normed_c2, bgpair2diffDist, p2z, deedpair2doublediffdist,
+                                                                        pval_threshold_basis = runconfig.cluster_threshold_pval, fcfc_threshold = runconfig.cluster_threshold_fcfc,
                                                                         take_median_ion=runconfig.take_median_ion, fcdiff_cutoff_clustermerge= runconfig.fcdiff_cutoff_clustermerge)
         protnodes.append(clustered_prot_node)
 
         if count_prots%100==0:
             LOGGER.info(f"checked {count_prots} of {len(prot2diffions.keys())} prots")
         count_prots+=1
-    
+
     if len(prot2missingval_diffions.keys())>0:
         LOGGER.info(f"start analysis of proteins w. completely missing values")
 
@@ -102,13 +102,13 @@ def analyze_condpair(*,runconfig, condpair):
             ions = prot2missingval_diffions.get(prot)
             protnode_missingval = aq_clust_missingval.create_protnode_from_missingval_ions(gene_name=prot,diffions=ions, normed_c1=normed_c1, normed_c2=normed_c2)
             protnodes_missingval.append(protnode_missingval)
-        
+
         LOGGER.info(f"finished missing value analysis")
 
     if runconfig.use_ml:
         ml_performance_dict = {}
 
-        #aq_class_stacked_frag.assign_predictability_scores_stacked(protein_nodes= protnodes, acquisition_info_df=None,results_dir=runconfig.results_dir, name = aqutils.get_condpairname(condpair)+"_fragions", 
+        #aq_class_stacked_frag.assign_predictability_scores_stacked(protein_nodes= protnodes, acquisition_info_df=None,results_dir=runconfig.results_dir, name = aqutils.get_condpairname(condpair)+"_fragions",
          #                           min_num_fragions=5, replace_nans=True, performance_metrics=ml_performance_dict, plot_predictor_performance=True)
         ml_successfull =aq_class_precursors.assign_predictability_scores(protein_nodes= protnodes, results_dir=runconfig.results_dir, name = aqutils.get_condpairname(condpair), ml_info_file=runconfig.ml_input_file,
                                         samples_used =c1_samples + c2_samples, min_num_precursors=3, prot_fc_cutoff=0, replace_nans=True, performance_metrics=ml_performance_dict, plot_predictor_performance=runconfig.runtime_plots)
@@ -159,10 +159,10 @@ def get_per_condition_dataframes(samples_c1, samples_c2, unnormed_df, minrep_bot
 
     if min_samples<2:
         raise Exception(f"condpair has not enough samples: c1:{len(samples_c1)} c2: {len(samples_c2)}, skipping")
-    
+
     if (minrep_either is not None) or ((minrep_c1 is not None) and (minrep_c2 is not None)): #minrep_both was set as default and should be overruled by minrep_either or minrep_c1 and minrep_c2
         minrep_both = None
-            
+
     if minrep_either is not None:
         minrep_either = np.min([get_minrep_for_cond(samples_c1, minrep_either), get_minrep_for_cond(samples_c2, minrep_either)])
         passes_minrep_c1 = unnormed_df.loc[:, samples_c1].notna().sum(axis=1) >= minrep_either
@@ -184,7 +184,7 @@ def get_per_condition_dataframes(samples_c1, samples_c2, unnormed_df, minrep_bot
         df_c2 = unnormed_df.loc[:, samples_c2].dropna(thresh=minrep_c2, axis=0)
         if (len(df_c1.index)<5) | (len(df_c2.index)<5):
             raise Exception(f"condpair has not enough data for processing c1: {len(df_c1.index)} c2: {len(df_c2.index)}, skipping")
-        
+
     if (minrep_both is None) and (minrep_either is None) and (minrep_c1 is None) and (minrep_c2 is None):
         raise Exception("no minrep set, please specify!")
 
@@ -200,13 +200,13 @@ def get_minrep_for_cond(c_samples, minrep):
         return num_samples
     else:
         return minrep
-    
 
-    
+
+
 
 def write_out_tables(condpair_node, runconfig):
     condpair = condpair_node.name
-    
+
     res_df = aq_tablewriter_protein.TableFromNodeCreator(condpair_node, node_type = "gene", min_num_peptides = runconfig.minpep, annotation_file= getattr(runconfig, "annotation_file", None)).results_df
     has_sequence_nodes = check_if_has_sequence_nodes(condpair_node)
     if has_sequence_nodes:
@@ -235,14 +235,17 @@ def write_out_tables(condpair_node, runconfig):
         res_df.to_csv(f"{runconfig.results_dir}/{aqutils.get_condpairname(condpair)}.results.tsv", sep = "\t", index=None)
         if has_sequence_nodes:
             pep_df.to_csv(f"{runconfig.results_dir}/{aqutils.get_condpairname(condpair)}.results.seq.tsv", sep = "\t", index=None)
-        
+
         if has_precursor_nodes:
             prec_df.to_csv(f"{runconfig.results_dir}/{aqutils.get_condpairname(condpair)}.results.prec.tsv", sep = "\t", index=None)
-        
+
     return res_df, pep_df
 
 def check_if_has_sequence_nodes(condpair_node):
     return condpair_node.children[0].children[0].type == "seq"
 
 def check_if_has_precursor_nodes(condpair_node):
-    return condpair_node.children[0].children[0].children[0].children[0].type == "mod_seq_charge"
\ No newline at end of file
+    try:
+        return condpair_node.children[0].children[0].children[0].children[0].type == "mod_seq_charge"
+    except:
+        return False
diff --git a/alphaquant/run_pipeline.py b/alphaquant/run_pipeline.py
index 29cc6976..8a34614c 100644
--- a/alphaquant/run_pipeline.py
+++ b/alphaquant/run_pipeline.py
@@ -122,22 +122,26 @@ def run_pipeline(input_file: str,
     if samplemap_df is None:
         samplemap_df = aq_diffquant_utils.load_samplemap(samplemap_file)
 
-    input_type, _, _ = config_dict_loader.get_input_type_and_config_dict(input_file_original, input_type_to_use)
+    input_type, config_dict, _ = config_dict_loader.get_input_type_and_config_dict(input_file_original, input_type_to_use)
     annotation_file = load_annotation_file(input_file_original, input_type, annotation_columns)
+    use_ml = check_if_table_supports_ml(config_dict)
 
     if perform_ptm_mapping:
         if modification_type is None:
             raise Exception("modification_type is None, but perform_ptm_mapping is True. Please set perform_ptm_mapping to False or specify modification_type.")
         input_file_reformat = load_ptm_input_file(input_file = input_file_original, input_type_to_use = "spectronaut_ptm_fragion", results_dir = results_dir, samplemap_df = samplemap_df, modification_type = modification_type, organism = organism)
-        ml_input_file = load_ml_info_file(input_file_original, input_type, modification_type)
+        if use_ml:
+            ml_input_file = load_ml_info_file(input_file_original, input_type, modification_type)
 
     elif "fragment_precursorfiltered.matrix" in input_file_original:
         alphadia_tableprocessor = aq_table_alphadiareader.AlphaDIAFragTableProcessor(input_file_original)
         input_file_reformat = alphadia_tableprocessor.input_file_reformat
-        ml_input_file = alphadia_tableprocessor.ml_info_file
+        if use_ml:
+            ml_input_file = alphadia_tableprocessor.ml_info_file
     else:
         input_file_reformat = load_input_file(input_file_original, input_type)
-        ml_input_file = load_ml_info_file(input_file_original, input_type)
+        if use_ml:
+            ml_input_file = load_ml_info_file(input_file_original, input_type)
 
     if peptides_to_exclude_file is not None:
         remove_peptides_to_exclude_from_input_file(input_file_reformat, peptides_to_exclude_file)
@@ -233,6 +237,9 @@ def load_annotation_file(input_file, input_type, annotation_columns):
     else:
         return aq_tablewriter_misc.AnnotationFileCreator(input_file, input_type, annotation_columns).annotation_filename
 
+def check_if_table_supports_ml(config_dict):
+    return config_dict["format"] == "longtable"
+
 def load_ml_info_file(input_file, input_type, modification_type = None):
     ml_info_filename = aq_utils.get_progress_folder_filename(input_file, f".ml_info_table.tsv")
     if os.path.exists(ml_info_filename):#in case there already is a reformatted file, we don't need to reformat it again
diff --git a/alphaquant/ui/dashboard_parts_run_pipeline.py b/alphaquant/ui/dashboard_parts_run_pipeline.py
index 1f7d88bd..4a58a1ce 100644
--- a/alphaquant/ui/dashboard_parts_run_pipeline.py
+++ b/alphaquant/ui/dashboard_parts_run_pipeline.py
@@ -18,9 +18,10 @@
 import alphaquant.ui.dashboard_parts_plots_proteoforms as dashboad_parts_plots_proteoforms
 import alphaquant.ui.gui as gui
 import alphaquant.ui.gui_textfields as gui_textfields
+import alphaquant.utils.reader_utils as aq_reader_utils
 
 import alphabase.quantification.quant_reader.config_dict_loader as config_dict_loader
-config_dict_loader.INTABLE_CONFIG = os.path.join(pathlib.Path(__file__).parent.absolute(), "../config/quant_reader_config_lightweight.yaml")
+config_dict_loader.INTABLE_CONFIG = os.path.join(pathlib.Path(__file__).parent.absolute(), "../config/quant_reader_config.yaml")
 # If using Plotly in Panel
 pn.extension('plotly')
 
@@ -886,13 +887,33 @@ def _import_sample_names(self):
 
 				input_file = self.path_analysis_file.value
 				_, config_dict, sep = config_dict_loader.get_input_type_and_config_dict(input_file)
-				sample_column = config_dict["sample_ID"]
-				sample_names = set()
-				for chunk in pd.read_csv(input_file, sep=sep, usecols=[sample_column], chunksize=400000):
-					sample_names.update(chunk[sample_column].unique())
-				self.sample_names = sample_names
+				if config_dict["format"] == "longtable":
+					sample_column = config_dict["sample_ID"]
+					sample_names = set()
+
+					for chunk in aq_reader_utils.read_file(input_file, sep=sep, usecols=[sample_column], chunksize=400000):
+						sample_names.update(chunk[sample_column].unique())
+					self.sample_names = sample_names
+				elif config_dict["format"] == "widetable":
+					# Read the headers first to identify sample columns
+					headers = aq_reader_utils.read_file(input_file, sep=sep, nrows=0).columns.tolist()
+
+					quant_pre_or_suffix = config_dict.get("quant_pre_or_suffix")
+					# Filter headers to find those with the prefix or suffix
+					sample_columns = [
+						col for col in headers if (
+							col.startswith(quant_pre_or_suffix) or
+							col.endswith(quant_pre_or_suffix)
+						)
+					]
+					self.sample_names = set([col.replace(quant_pre_or_suffix, '') for col in sample_columns])
+				else:
+					print("ERROR: Could not idenfity sample names in the input file.")
+					self.run_pipeline_error.object = "Could not idenfity sample names . Please check your input file."
+					self.run_pipeline_error.visible = True
 
 			except Exception as e:
+				print(f"Error importing data: {e}")
 				self.run_pipeline_error.object = f"Error importing data: {e}"
 				self.run_pipeline_error.visible = True
 			finally:
@@ -1127,6 +1148,7 @@ def _update_samplemap(self, event):
 				self.state.notify_subscribers('samplemap_df')
 
 			except Exception as e:
+				print(f"Error reading sample map: {str(e)}")
 				self.run_pipeline_error.object = f"Error reading sample map: {str(e)}"
 				self.run_pipeline_error.visible = True
 
@@ -1156,6 +1178,10 @@ def _generate_samplemap(self, event):
 				self.template_success_message.object = f"""Template has been generated. Please fill out the condition column in the table below.\nThe template has also been saved to
 				<code>{template_path}</code>\nif you prefer to edit it with Excel or other applications."""
 				self.template_success_message.visible = True
+		except Exception as e:
+			print(f"Error generating sample map: {str(e)}")
+			self.run_pipeline_error.object = f"Error generating sample map: {str(e)}"
+			self.run_pipeline_error.visible = True
 		finally:
 			# Hide loading indicators when done
 			self.loading_samples_indicator.visible = False
diff --git a/alphaquant/utils/reader_utils.py b/alphaquant/utils/reader_utils.py
index fadeefdb..fdff828b 100644
--- a/alphaquant/utils/reader_utils.py
+++ b/alphaquant/utils/reader_utils.py
@@ -4,9 +4,11 @@
 LOGGER = logging.getLogger(__name__)
 
 
-def read_file(file_path, decimal=".", usecols=None, chunksize=None, sep=None):
+def read_file(file_path, decimal=".", usecols=None, chunksize=None, sep=None, nrows=None):
     file_path = str(file_path)
     if ".parquet" in file_path:
+        if nrows is not None:
+            LOGGER.warning(f"nrows parameter is set, but not supported for parquet files. Ignoring nrows parameter.")
         return _read_parquet_file(file_path, usecols=usecols, chunksize=chunksize)
     else:
         if sep is None:
@@ -26,6 +28,7 @@ def read_file(file_path, decimal=".", usecols=None, chunksize=None, sep=None):
             usecols=usecols,
             encoding="latin1",
             chunksize=chunksize,
+            nrows=nrows,
         )
 
 
diff --git a/tests/e2e_tests_small/different_input_tables.ipynb b/tests/e2e_tests_small/different_input_tables.ipynb
new file mode 100644
index 00000000..68b4ddcf
--- /dev/null
+++ b/tests/e2e_tests_small/different_input_tables.ipynb
@@ -0,0 +1,82 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "TEST_FILE_DIR = \"../../test_data/quicktests/input_table_formats\"\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import alphabase.tools.data_downloader as ab_data_downloader\n",
+    "\n",
+    "downloader = ab_data_downloader.DataShareDownloader(\"https://datashare.biochem.mpg.de/s/UiyNiR9SlWXJ6qM\", output_dir=f\"{TEST_FILE_DIR}/..\")#this downloads the folder input_table_formats from the AlphaX data share into the specified path\n",
+    "\n",
+    "downloader.download()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import os\n",
+    "import alphaquant.run_pipeline as aq_run_pipeline\n",
+    "\n",
+    "input_files = [ \"diann.parquet\",\"spectronaut.frgions.large.tsv\", \"mq_peptides.txt\", \"mq_evidence.txt\", \"fragpipe.tsv\"]\n",
+    "samplemap_map = {\n",
+    "\t\"diann.parquet\": f\"{TEST_FILE_DIR}/samplemap_diann_parquet.tsv\",\n",
+    "\t\"spectronaut.frgions.large.tsv\": f\"{TEST_FILE_DIR}/samplemap_spectronaut.tsv\",\n",
+    "\t\"mq_peptides.txt\": f\"{TEST_FILE_DIR}/samplemap_maxquant.tsv\",\n",
+    "\t\"mq_evidence.txt\": f\"{TEST_FILE_DIR}/samplemap_maxquant.tsv\",\n",
+    "\t\"fragpipe.tsv\": f\"{TEST_FILE_DIR}/samplemap_fragpipe.tsv\"\n",
+    "}\n",
+    "results_dir_map = {\n",
+    "\t\"diann.parquet\": f\"{TEST_FILE_DIR}/results_diann_parquet\",\n",
+    "\t\"spectronaut.frgions.large.tsv\": f\"{TEST_FILE_DIR}/results_spectronaut\",\n",
+    "\t\"mq_peptides.txt\": f\"{TEST_FILE_DIR}/results_mq_peptides\",\n",
+    "\t\"mq_evidence.txt\": f\"{TEST_FILE_DIR}/results_mq_evidence\",\n",
+    "\t\"fragpipe.tsv\": f\"{TEST_FILE_DIR}/results_fragpipe\"\n",
+    "}\n",
+    "\n",
+    "for input_file in input_files:\n",
+    "\tif input_file ==\"fragpipe.tsv\":\n",
+    "\t\taq_run_pipeline.run_pipeline(input_file=os.path.join(TEST_FILE_DIR, input_file), samplemap_file=samplemap_map[input_file], results_dir=results_dir_map[input_file], reset_progress_folder=True, multicond_median_analysis=True)\n",
+    "\telse:\n",
+    "\t\taq_run_pipeline.run_pipeline(input_file=os.path.join(TEST_FILE_DIR, input_file), samplemap_file=samplemap_map[input_file], results_dir=results_dir_map[input_file], reset_progress_folder=True)\n",
+    "\n",
+    "\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "test",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/tests/run_e2e_tests_quick.sh b/tests/run_e2e_tests_quick.sh
index 2daf64e3..0c6014ab 100755
--- a/tests/run_e2e_tests_quick.sh
+++ b/tests/run_e2e_tests_quick.sh
@@ -3,4 +3,5 @@ conda activate alphaquant
 echo "Running e2e tests quick"
 pytest --nbmake e2e_tests_small/mixed_species.ipynb
 pytest --nbmake e2e_tests_small/phospho.ipynb
+pytest --nbmake e2e_tests_small/different_input_tables.ipynb
 conda deactivate


Reviews:
user: login: mschwoer
body: apparently, the code-review but also prefers ruff-formatted code :-p
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T14:48:52Z
user: login: ammarcsj
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-14T15:39:08Z
user: login: ammarcsj
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-14T15:45:10Z

Review comments:
diff_hunk: @@ -886,13 +887,33 @@ def _import_sample_names(self):
 
 				input_file = self.path_analysis_file.value
 				_, config_dict, sep = config_dict_loader.get_input_type_and_config_dict(input_file)
-				sample_column = config_dict["sample_ID"]
-				sample_names = set()
-				for chunk in pd.read_csv(input_file, sep=sep, usecols=[sample_column], chunksize=400000):
-					sample_names.update(chunk[sample_column].unique())
-				self.sample_names = sample_names
+				if config_dict["format"] == "longtable":
+					sample_column = config_dict["sample_ID"]
+					sample_names = set()
+
+					for chunk in aq_reader_utils.read_file(input_file, sep=sep, usecols=[sample_column], chunksize=400000):
+						sample_names.update(chunk[sample_column].unique())
+					self.sample_names = sample_names
+				elif config_dict["format"] == "widetable":
+					# Read the headers first to identify sample columns
+					headers = aq_reader_utils.read_file(input_file, sep=sep, nrows=0).columns.tolist()
+
+					quant_pre_or_suffix = config_dict.get("quant_pre_or_suffix")
+					# Filter headers to find those with the prefix or suffix
+					sample_columns = [
+						col for col in headers if (
+							col.startswith(quant_pre_or_suffix) or
+							col.endswith(quant_pre_or_suffix)
+						)
+					]
+					self.sample_names = set([col.replace(quant_pre_or_suffix, '') for col in sample_columns])
+				else:
+					print("ERROR: Could not idenfity sample names in the input file.")
path: alphaquant/ui/dashboard_parts_run_pipeline.py
user: login: mschwoer
body: isn't this rather `"unknown 'format' in config: {config_dict["format"]}}` ?
created_at: 2025-03-14T14:45:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -122,22 +122,26 @@ def run_pipeline(input_file: str,
     if samplemap_df is None:
         samplemap_df = aq_diffquant_utils.load_samplemap(samplemap_file)
 
-    input_type, _, _ = config_dict_loader.get_input_type_and_config_dict(input_file_original, input_type_to_use)
+    input_type, config_dict, _ = config_dict_loader.get_input_type_and_config_dict(input_file_original, input_type_to_use)
     annotation_file = load_annotation_file(input_file_original, input_type, annotation_columns)
+    use_ml = check_if_table_supports_ml(config_dict)
 
     if perform_ptm_mapping:
         if modification_type is None:
             raise Exception("modification_type is None, but perform_ptm_mapping is True. Please set perform_ptm_mapping to False or specify modification_type.")
         input_file_reformat = load_ptm_input_file(input_file = input_file_original, input_type_to_use = "spectronaut_ptm_fragion", results_dir = results_dir, samplemap_df = samplemap_df, modification_type = modification_type, organism = organism)
-        ml_input_file = load_ml_info_file(input_file_original, input_type, modification_type)
+        if use_ml:
+            ml_input_file = load_ml_info_file(input_file_original, input_type, modification_type)
 
     elif "fragment_precursorfiltered.matrix" in input_file_original:
         alphadia_tableprocessor = aq_table_alphadiareader.AlphaDIAFragTableProcessor(input_file_original)
         input_file_reformat = alphadia_tableprocessor.input_file_reformat
-        ml_input_file = alphadia_tableprocessor.ml_info_file
+        if use_ml:
+            ml_input_file = alphadia_tableprocessor.ml_info_file
     else:
         input_file_reformat = load_input_file(input_file_original, input_type)
-        ml_input_file = load_ml_info_file(input_file_original, input_type)
+        if use_ml:
+            ml_input_file = load_ml_info_file(input_file_original, input_type)
path: alphaquant/run_pipeline.py
user: login: mschwoer
body: `ml_input_file` seems not to be used downstream?
created_at: 2025-03-14T14:48:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -886,13 +887,33 @@ def _import_sample_names(self):
 
 				input_file = self.path_analysis_file.value
 				_, config_dict, sep = config_dict_loader.get_input_type_and_config_dict(input_file)
-				sample_column = config_dict["sample_ID"]
-				sample_names = set()
-				for chunk in pd.read_csv(input_file, sep=sep, usecols=[sample_column], chunksize=400000):
-					sample_names.update(chunk[sample_column].unique())
-				self.sample_names = sample_names
+				if config_dict["format"] == "longtable":
+					sample_column = config_dict["sample_ID"]
+					sample_names = set()
+
+					for chunk in aq_reader_utils.read_file(input_file, sep=sep, usecols=[sample_column], chunksize=400000):
+						sample_names.update(chunk[sample_column].unique())
+					self.sample_names = sample_names
+				elif config_dict["format"] == "widetable":
+					# Read the headers first to identify sample columns
+					headers = aq_reader_utils.read_file(input_file, sep=sep, nrows=0).columns.tolist()
+
+					quant_pre_or_suffix = config_dict.get("quant_pre_or_suffix")
+					# Filter headers to find those with the prefix or suffix
+					sample_columns = [
+						col for col in headers if (
+							col.startswith(quant_pre_or_suffix) or
+							col.endswith(quant_pre_or_suffix)
+						)
+					]
+					self.sample_names = set([col.replace(quant_pre_or_suffix, '') for col in sample_columns])
+				else:
+					print("ERROR: Could not idenfity sample names in the input file.")
path: alphaquant/ui/dashboard_parts_run_pipeline.py
user: login: ammarcsj
body: 👍 
created_at: 2025-03-14T15:39:08Z
author_association: MEMBER
diff_hunk: @@ -122,22 +122,26 @@ def run_pipeline(input_file: str,
     if samplemap_df is None:
         samplemap_df = aq_diffquant_utils.load_samplemap(samplemap_file)
 
-    input_type, _, _ = config_dict_loader.get_input_type_and_config_dict(input_file_original, input_type_to_use)
+    input_type, config_dict, _ = config_dict_loader.get_input_type_and_config_dict(input_file_original, input_type_to_use)
     annotation_file = load_annotation_file(input_file_original, input_type, annotation_columns)
+    use_ml = check_if_table_supports_ml(config_dict)
 
     if perform_ptm_mapping:
         if modification_type is None:
             raise Exception("modification_type is None, but perform_ptm_mapping is True. Please set perform_ptm_mapping to False or specify modification_type.")
         input_file_reformat = load_ptm_input_file(input_file = input_file_original, input_type_to_use = "spectronaut_ptm_fragion", results_dir = results_dir, samplemap_df = samplemap_df, modification_type = modification_type, organism = organism)
-        ml_input_file = load_ml_info_file(input_file_original, input_type, modification_type)
+        if use_ml:
+            ml_input_file = load_ml_info_file(input_file_original, input_type, modification_type)
 
     elif "fragment_precursorfiltered.matrix" in input_file_original:
         alphadia_tableprocessor = aq_table_alphadiareader.AlphaDIAFragTableProcessor(input_file_original)
         input_file_reformat = alphadia_tableprocessor.input_file_reformat
-        ml_input_file = alphadia_tableprocessor.ml_info_file
+        if use_ml:
+            ml_input_file = alphadia_tableprocessor.ml_info_file
     else:
         input_file_reformat = load_input_file(input_file_original, input_type)
-        ml_input_file = load_ml_info_file(input_file_original, input_type)
+        if use_ml:
+            ml_input_file = load_ml_info_file(input_file_original, input_type)
path: alphaquant/run_pipeline.py
user: login: ammarcsj
body: yes, so all the variables are stored in a "runconfig" class and then given over to the actual processing function
created_at: 2025-03-14T15:45:10Z
author_association: MEMBER

PR comments:
user: login: github-actions[bot]
created_at: 2025-03-14T14:20:00Z
body: The following feedback could not be added to specific lines, but still contains valuable information:

```
Looking at the code changes, I've identified several issues that need to be addressed:

[
  {
    "change_id": 1,
    "file_name": "./alphaquant/diffquant/condpair_analysis.py",
    "start_line": 237,
    "end_line": 238,
    "proposed_code": "def check_if_has_precursor_nodes(condpair_node):\n    try:\n        return condpair_node.children[0].children[0].children[0].children[0].type == "mod_seq_charge"\n    except:\n        return False",
    "comment": "The original code doesn't handle cases where the node structure might not have the expected depth. This can cause IndexError exceptions when processing certain data types. The try-except block provides a safer way to check for precursor nodes."
  },
  {
    "change_id": 2,
    "file_name": "./alphaquant/config/quant_reader_config.yaml",
    "start_line": 145,
    "end_line": 154,
    "proposed_code": "maxquant_evidence:\n  format: longtable\n  sample_ID: Experiment\n  quant_ID:\n    precursor: Intensity\n  protein_cols:\n   - Gene names\n  ion_hierarchy:\n    precursor:\n      order: [SEQ, MOD, CHARGE]\n      mapping:\n        SEQ:\n          - Sequence\n        MOD:\n          - Mass\n        CHARGE:\n          - Charge\n  filters:\n    reverse:\n      param: Reverse\n      comparator: "!="\n      value: "+"\n    contaminant:\n      param: Potential contaminant\n      comparator: "!="\n      value: "+"\n  ml_level: SEQ\n  use_iontree: False",
    "comment": "The MaxQuant evidence configuration was updated to use a hierarchical structure for ion representation rather than a flat list of ion_cols. This improves compatibility with the AlphaQuant pipeline and provides better support for proper precursor quantification. The filters for reverse and contaminant proteins were also added to improve data quality."
  },
  {
    "change_id": 3,
    "file_name": "./alphaquant/run_pipeline.py",
    "start_line": 124,
    "end_line": 125,
    "proposed_code": "    input_type, config_dict, _ = config_dict_loader.get_input_type_and_config_dict(input_file_original, input_type_to_use)\n    annotation_file = load_annotation_file(input_file_original, input_type, annotation_columns)\n    use_ml = check_if_table_supports_ml(config_dict)",
    "comment": "The original code doesn't capture the config_dict from the return value, which is needed later to check if ML is supported. Also, a new use_ml variable is introduced to track whether ML should be used for the current input type."
  },
  {
    "change_id": 4,
    "file_name": "./alphaquant/run_pipeline.py",
    "start_line": 240,
    "end_line": 240,
    "proposed_code": "def check_if_table_supports_ml(config_dict):\n    return config_dict["format"] == "longtable"",
    "comment": "Added a helper function to check if machine learning is supported for the current input format. Currently, ML is only supported for longtable formats."
  },
  {
    "change_id": 5,
    "file_name": "./alphaquant/run_pipeline.py",
    "start_line": 130,
    "end_line": 136,
    "proposed_code": "        input_file_reformat = load_ptm_input_file(input_file = input_file_original, input_type_to_use = "spectronaut_ptm_fragion", results_dir = results_dir, samplemap_df = samplemap_df, modification_type = modification_type, organism = organism)\n        if use_ml:\n            ml_input_file = load_ml_info_file(input_file_original, input_type, modification_type)\n\n    elif "fragment_precursorfiltered.matrix" in input_file_original:\n        alphadia_tableprocessor = aq_table_alphadiareader.AlphaDIAFragTableProcessor(input_file_original)\n        input_file_reformat = alphadia_tableprocessor.input_file_reformat\n        if use_ml:\n            ml_input_file = alphadia_tableprocessor.ml_info_file",
    "comment": "Modified the code to check if ML is supported before loading ML information files. This prevents errors when trying to use ML with formats that don't support it."
  },
  {
    "change_id": 6,
    "file_name": "./alphaquant/run_pipeline.py",
    "start_line": 142,
    "end_line": 143,
    "proposed_code": "        input_file_reformat = load_input_file(input_file_original, input_type)\n        if use_ml:\n            ml_input_file = load_ml_info_file(input_file_original, input_type)",
    "comment": "Added the same ML support check in the general case, ensuring ML info is only loaded when the format supports it."
  },
  {
    "change_id": 7,
    "file_name": "./alphaquant/utils/reader_utils.py",
    "start_line": 7,
    "end_line": 10,
    "proposed_code": "def read_file(file_path, decimal=".", usecols=None, chunksize=None, sep=None, nrows=None):\n    file_path = str(file_path)\n    if ".parquet" in file_path:\n        if nrows is not None:\n            LOGGER.warning(f"nrows parameter is set, but not supported for parquet files. Ignoring nrows parameter.")\n        return _read_parquet_file(file_path, usecols=usecols, chunksize=chunksize)",
    "comment": "Added 'nrows' parameter to the read_file function to allow reading only a subset of rows. This is helpful for examining large files or for testing. The function also adds a warning when nrows is used with parquet files since that format doesn't directly support it."
  },
  {
    "change_id": 8,
    "file_name": "./alphaquant/utils/reader_utils.py",
    "start_line": 28,
    "end_line": 30,
    "proposed_code": "            usecols=usecols,\n            encoding="latin1",\n            chunksize=chunksize,\n            nrows=nrows,\n        )",
    "comment": "Added the nrows parameter to the pandas.read_csv call to enable partial file reading."
  },
  {
    "change_id": 9,
    "file_name": "./alphaquant/ui/dashboard_parts_run_pipeline.py",
    "start_line": 886,
    "end_line": 903,
    "proposed_code": "				input_file = self.path_analysis_file.value\n				_, config_dict, sep = config_dict_loader.get_input_type_and_config_dict(input_file)\n				if config_dict["format"] == "longtable":\n					sample_column = config_dict["sample_ID"]\n					sample_names = set()\n\n					for chunk in aq_reader_utils.read_file(input_file, sep=sep, usecols=[sample_column], chunksize=400000):\n						sample_names.update(chunk[sample_column].unique())\n					self.sample_names = sample_names\n				elif config_dict["format"] == "widetable":\n					# Read the headers first to identify sample columns\n					headers = aq_reader_utils.read_file(input_file, sep=sep, nrows=0).columns.tolist()\n\n					quant_pre_or_suffix = config_dict.get("quant_pre_or_suffix")\n					# Filter headers to find those with the prefix or suffix\n					sample_columns = [\n						col for col in headers if (\n							col.startswith(quant_pre_or_suffix) or\n							col.endswith(quant_pre_or_suffix)\n						)\n					]\n					self.sample_names = set([col.replace(quant_pre_or_suffix, "") for col in sample_columns])\n				else:\n					print("ERROR: Could not idenfity sample names in the input file.")\n					self.run_pipeline_error.object = "Could not idenfity sample names . Please check your input file."\n					self.run_pipeline_error.visible = True",
    "comment": "Added support for wide format tables in the UI sample detection code. Previously it only worked with longtable formats. The code now checks the format type and uses different methods to extract sample names based on the table format."
  }
]
```
author_association: NONE
user: login: github-actions[bot]
created_at: 2025-03-14T14:20:02Z
body: Number of tokens: input_tokens=42831 output_tokens=2417 max_tokens=4096
review_instructions=''
config={}
thinking: ```
[]
```
author_association: NONE