Upload esmbind-validation-notebook.ipynb

esmbind-validation-notebook.ipynb

@@ -0,0 +1 @@
+{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install transformers -q\n!pip install accelerate -q\n!pip install peft -q\n!pip install datasets -q","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing ESMBind (ESMB) for Protein Binding Residue Prediction\n\nThis notebook is meant to test out ESM-2 LoRA models on the datasets found [here](https://github.com/hamzagamouh/pt-lm-gnn/tree/main/datasets/yu_merged) for the paper [Hybrid protein-ligand binding residue prediction with protein\nlanguage models: Does the structure matter?](https://www.biorxiv.org/content/10.1101/2023.08.11.553028v1). The models referenced in the paper are GCN, GAT, and ensemble structural models trained on PDB sequences to predict binding residues. They are the best performing models that could be found as of 17/09/23. You will need to download the datasets you want to test out from the github above and provide the file path in the code below. For your convenience, some of the performance metrics are provided below:\n\n```python\nTrain metrics: \nAverage Accuracy: 0.9169\nAverage Precision: 0.1416\nAverage Recall: 0.2769\nAverage F1 Score: 0.1743\nAverage AUC: 0.6088\nAverage MCC: 0.1522\n\nTest metrics: \nAverage Accuracy: 0.9199\nAverage Precision: 0.1307\nAverage Recall: 0.2750\nAverage F1 Score: 0.1677\nAverage AUC: 0.6081\nAverage MCC: 0.1474\n```\n\nYet the model does not seem to generalize well to the PDB datasets. ","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# Load the dataset\ndata_df = pd.read_csv(\"/kaggle/input/binding-sites-struct/CA_Training.txt\", delimiter=';')\n\n# Display the first few rows of the dataframe to understand its structure\ndata_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a function to convert binding residues to binary labels\ndef binding_residues_to_labels(row):\n    sequence = row['sequence']\n    binding_residues = row['binding_residues']\n\n    # Initialize a list with zeros\n    labels = [0] * len(sequence)\n\n    # If binding_residues is not NaN, mark the binding residues in the labels list with 1\n    if isinstance(binding_residues, str):\n        # Get the indices of the binding residues\n        binding_residues_indices = [int(residue[1:]) - 1 for residue in binding_residues.split()]\n\n        # Mark the binding residues in the labels list with 1\n        for idx in binding_residues_indices:\n            if idx < len(labels):\n                labels[idx] = 1\n\n    return labels\n\n# Apply the function to each row in the DataFrame to get the binary labels\ndata_df['binding_labels'] = data_df.apply(binding_residues_to_labels, axis=1)\n\n# Display the first few rows of the DataFrame\ndata_df.head()\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the maximum chunk size\nMAX_CHUNK_SIZE = 900\n\n# Function to segment sequences and labels into chunks of size <= 1022\ndef segment_into_chunks(row):\n    sequence = row['sequence']\n    labels = row['binding_labels']\n\n    # Segment the sequence and labels into chunks of size <= 1022\n    sequence_chunks = [sequence[i:i+MAX_CHUNK_SIZE] for i in range(0, len(sequence), MAX_CHUNK_SIZE)]\n    label_chunks = [labels[i:i+MAX_CHUNK_SIZE] for i in range(0, len(labels), MAX_CHUNK_SIZE)]\n\n    return sequence_chunks, label_chunks\n\n# Apply the function to each row in the DataFrame to get the segmented sequences and labels\ndata_df['sequence_chunks'] = None\ndata_df['label_chunks'] = None\nfor idx, row in data_df.iterrows():\n    data_df.at[idx, 'sequence_chunks'], data_df.at[idx, 'label_chunks'] = segment_into_chunks(row)\n\n# Display the first few rows of the DataFrame\ndata_df[['pdb_id', 'chain_id', 'sequence_chunks', 'label_chunks']].head()\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoModelForTokenClassification, AutoTokenizer\nfrom peft import PeftModel\nimport torch\n\ndef get_predictions(protein_sequence):\n    # Path to the saved LoRA model\n    model_path = \"AmelieSchreiber/esm2_t12_35M_lora_binding_sites_1111K_cp1\"\n    # ESM2 base model\n    base_model_path = \"facebook/esm2_t12_35M_UR50D\"\n\n    # Load the model\n    base_model = AutoModelForTokenClassification.from_pretrained(base_model_path)\n    loaded_model = PeftModel.from_pretrained(base_model, model_path)\n\n    # Ensure the model is in evaluation mode\n    loaded_model.eval()\n\n    # Load the tokenizer\n    loaded_tokenizer = AutoTokenizer.from_pretrained(base_model_path)\n\n    # Tokenize the sequence\n    inputs = loaded_tokenizer(protein_sequence, return_tensors=\"pt\", truncation=True, max_length=1024, padding='max_length')\n\n    # Run the model\n    with torch.no_grad():\n        logits = loaded_model(**inputs).logits\n\n    # Get predictions\n    tokens = loaded_tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"][0])  # Convert input ids back to tokens\n    predictions = torch.argmax(logits, dim=2)[0].numpy()\n\n    # Define labels\n    id2label = {\n        0: \"No binding site\",\n        1: \"Binding site\"\n    }\n\n    # Convert predictions to binary labels (1 for binding site, 0 otherwise)\n    special_tokens = ['<cls>', '<pad>', '<eos>', '<unk>', '.', '-', '<null_1>', '<mask>']\n    binary_predictions = [1 if id2label[pred] == \"Binding site\" else 0 for token, pred in zip(tokens, predictions) if token not in special_tokens]\n\n    return binary_predictions\n\n# Use the function to get predictions for a test sequence\ntest_sequence = \"MAVPETRPNHTIYINNLNEKIKKDELKKSLHAIFSRFGQILDILVSRSLKMRGQAFVIFKEVSSATNALRSMQGFPFYDKPMRIQYAKTDSDIIAKMKGT\"\nprint(get_predictions(test_sequence))\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoModelForTokenClassification, AutoTokenizer\nfrom peft import PeftModel\nimport torch\nfrom tqdm import tqdm\nfrom tqdm.notebook import tqdm as tqdm_notebook  # for notebook-compatible progress bars\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score, matthews_corrcoef\n\n# Check if a GPU is available and if not, use a CPU\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ndef get_predictions(protein_sequence, loaded_model, loaded_tokenizer):\n    # Tokenize the sequence\n    inputs = loaded_tokenizer(protein_sequence, return_tensors=\"pt\", truncation=True, max_length=1000, padding='max_length')\n\n    # Move the inputs to the GPU if available\n    inputs = {name: tensor.to(device) for name, tensor in inputs.items()}\n\n    # Run the model\n    with torch.no_grad():\n        logits = loaded_model(**inputs).logits\n\n    # Get predictions\n    tokens = loaded_tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"][0].cpu())  # Convert input ids back to tokens\n    predictions = torch.argmax(logits, dim=2)[0].cpu().numpy()  # Move logits to CPU before converting to numpy\n\n    # Define labels\n    id2label = {\n        0: \"No binding site\",\n        1: \"Binding site\"\n    }\n\n    # Convert predictions to binary labels (1 for binding site, 0 otherwise)\n    special_tokens = ['<cls>', '<pad>', '<eos>', '<unk>', '.', '-', '<null_1>', '<mask>']\n    binary_predictions = [1 if id2label[pred] == \"Binding site\" else 0 for token, pred in zip(tokens, predictions) if token not in special_tokens]\n\n    return binary_predictions\n\n# Load the model and tokenizer\nbase_model_path = \"facebook/esm2_t12_35M_UR50D\"\nbase_model = AutoModelForTokenClassification.from_pretrained(base_model_path)\nlora_model_path = \"AmelieSchreiber/esm2_t12_35M_lora_binding_sites_1111K_cp1\"\nloaded_model = PeftModel.from_pretrained(base_model, lora_model_path)\nloaded_model.eval()\nloaded_model.to(device)  # Move the model to the GPU\nloaded_tokenizer = AutoTokenizer.from_pretrained(base_model_path)\n\n# Set up tqdm for pandas\ntqdm.pandas(desc=\"Processing rows\")\n\ndef get_chunk_predictions(row):\n    global loaded_model, loaded_tokenizer\n    sequence_chunks = row['sequence_chunks']\n    predictions = [get_predictions(chunk, loaded_model, loaded_tokenizer) for chunk in sequence_chunks]\n    return predictions\n\n# Apply the function with a progress bar using tqdm.pandas\ndata_df['predictions_chunks'] = data_df.progress_apply(get_chunk_predictions, axis=1)\n\n# Flatten the lists of labels and predictions to calculate metrics\ntrue_labels_flat = [label for sublist in data_df['label_chunks'].tolist() for subsublist in sublist for label in subsublist]\npredictions_flat = [label for sublist in data_df['predictions_chunks'].tolist() for subsublist in sublist for label in subsublist]\n\n# Calculate the metrics\naccuracy = accuracy_score(true_labels_flat, predictions_flat)\nprecision = precision_score(true_labels_flat, predictions_flat)\nrecall = recall_score(true_labels_flat, predictions_flat)\nf1 = f1_score(true_labels_flat, predictions_flat)\nauc = roc_auc_score(true_labels_flat, predictions_flat)\nmcc = matthews_corrcoef(true_labels_flat, predictions_flat)\n\n# Print the metrics\nprint(f'Accuracy: {accuracy:.4f}')\nprint(f'Precision: {precision:.4f}')\nprint(f'Recall: {recall:.4f}')\nprint(f'F1 Score: {f1:.4f}')\nprint(f'AUC: {auc:.4f}')\nprint(f'MCC: {mcc:.4f}')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ADP_Training\n```python\nAccuracy: 0.9212\nPrecision: 0.2112\nRecall: 0.4052\nF1 Score: 0.2777\nAUC: 0.6732\nMCC: 0.2547\n```\n\n### AMP_Training\n```python\nAccuracy: 0.9054\nPrecision: 0.1444\nRecall: 0.3481\nF1 Score: 0.2041\nAUC: 0.6368\nMCC: 0.1809\n```\n\n### ATP_Training\n```python\nAccuracy: 0.9237\nPrecision: 0.2270\nRecall: 0.3754\nF1 Score: 0.2829\nAUC: 0.6610\nMCC: 0.2539\n```\n\n### CA_Training\n```python\nAccuracy: 0.9040\nPrecision: 0.0367\nRecall: 0.1866\nF1 Score: 0.0613\nAUC: 0.5514\nMCC: 0.0473\n```\n\n### DNA_Training\n```python\nAccuracy: 0.8764\nPrecision: 0.1943\nRecall: 0.1549\nF1 Score: 0.1724\nAUC: 0.5484\nMCC: 0.1073\n```\n\n### FE_Training\n```python\nAccuracy: 0.9286\nPrecision: 0.0584\nRecall: 0.2298\nF1 Score: 0.0932\nAUC: 0.5849\nMCC: 0.0877\n```\n\n### GDP_Training\n```python\nAccuracy: 0.9154\nPrecision: 0.2475\nRecall: 0.5395\nF1 Score: 0.3393\nAUC: 0.7353\nMCC: 0.3270\n```\n\n### GTP_Training\n```python\nAccuracy: 0.9220\nPrecision: 0.2032\nRecall: 0.4443\nF1 Score: 0.2789\nAUC: 0.6915\nMCC: 0.2646\n```\n\n### HEME_Training\n```python\nAccuracy: 0.8807\nPrecision: 0.1986\nRecall: 0.1564\nF1 Score: 0.1750\nAUC: 0.5504\nMCC: 0.1126\n```\n\n### MG_Training\n\n```python\nAccuracy: 0.9171\nPrecision: 0.0417\nRecall: 0.3013\nF1 Score: 0.0733\nAUC: 0.6126\nMCC: 0.0868\n```\n\n### MN_Training\n\n```python\nAccuracy: 0.9190\nPrecision: 0.0579\nRecall: 0.3382\nF1 Score: 0.0989\nAUC: 0.6325\nMCC: 0.1133\n```\n\n### ZN_Training\n\n```python\nAccuracy: 0.9171\nPrecision: 0.0381\nRecall: 0.1915\nF1 Score: 0.0636\nAUC: 0.5597\nMCC: 0.0550\n```\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport torch\nfrom transformers import AutoModelForTokenClassification, AutoTokenizer\nfrom peft import PeftModel\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score, matthews_corrcoef\nfrom tqdm import tqdm\nfrom tqdm.notebook import tqdm as tqdm_notebook  # for notebook-compatible progress bars\nimport glob\nimport os\n\n# Function to convert binding residues to binary labels\ndef binding_residues_to_labels(row):\n    sequence = row['sequence']\n    binding_residues = row['binding_residues']\n    labels = [0] * len(sequence)\n    if isinstance(binding_residues, str):\n        binding_residues_indices = [int(residue[1:]) - 1 for residue in binding_residues.split()]\n        for idx in binding_residues_indices:\n            if idx < len(labels):\n                labels[idx] = 1\n    return labels\n\n# Function to segment sequences and labels into chunks of size <= 1022\ndef segment_into_chunks(row):\n    MAX_CHUNK_SIZE = 900\n    sequence = row['sequence']\n    labels = row['binding_labels']\n    sequence_chunks = [sequence[i:i+MAX_CHUNK_SIZE] for i in range(0, len(sequence), MAX_CHUNK_SIZE)]\n    label_chunks = [labels[i:i+MAX_CHUNK_SIZE] for i in range(0, len(labels), MAX_CHUNK_SIZE)]\n    return sequence_chunks, label_chunks\n\ndef get_predictions(protein_sequence, loaded_model, loaded_tokenizer):\n    inputs = loaded_tokenizer(protein_sequence, return_tensors=\"pt\", truncation=True, max_length=1000, padding='max_length')\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    inputs = {name: tensor.to(device) for name, tensor in inputs.items()}\n    with torch.no_grad():\n        logits = loaded_model(**inputs).logits\n    tokens = loaded_tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"][0].cpu())\n    predictions = torch.argmax(logits, dim=2)[0].cpu().numpy()\n    id2label = {0: \"No binding site\", 1: \"Binding site\"}\n    special_tokens = ['<cls>', '<pad>', '<eos>', '<unk>', '.', '-', '<null_1>', '<mask>']\n    binary_predictions = [1 if id2label[pred] == \"Binding site\" else 0 for token, pred in zip(tokens, predictions) if token not in special_tokens]\n    return binary_predictions\n\ndef get_chunk_predictions(row):\n    global loaded_model, loaded_tokenizer\n    sequence_chunks = row['sequence_chunks']\n    predictions = [get_predictions(chunk, loaded_model, loaded_tokenizer) for chunk in sequence_chunks]\n    return predictions\n\n# Set up tqdm for pandas\ntqdm.pandas(desc=\"Processing rows\")\n\n# Define the directory containing the datasets\ndatasets_dir = \"/kaggle/input/struct-validation\"\n\n# Use glob to get a list of all datasets in the directory\ndatasets = glob.glob(os.path.join(datasets_dir, \"*.txt\"))\n\n# Load the model and tokenizer\nbase_model_path = \"facebook/esm2_t12_35M_UR50D\"\nbase_model = AutoModelForTokenClassification.from_pretrained(base_model_path)\nlora_model_path = \"AmelieSchreiber/esm2_t12_35M_lora_binding_sites_1111K_cp1\"\nloaded_model = PeftModel.from_pretrained(base_model, lora_model_path)\nloaded_model.eval()\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nloaded_model.to(device)  # Move the model to the GPU\nloaded_tokenizer = AutoTokenizer.from_pretrained(base_model_path)\n\n# Now initialize lists to collect metrics\naccuracy_list = []\nprecision_list = []\nrecall_list = []\nf1_list = []\nauc_list = []\nmcc_list = []\n\n# Loop through each dataset and evaluate the model\nfor dataset_path in datasets:\n    # Load the dataset\n    data_df = pd.read_csv(dataset_path, delimiter=';')\n\n    # Apply the function to each row in the DataFrame to get the binary labels\n    data_df['binding_labels'] = data_df.apply(binding_residues_to_labels, axis=1)\n\n    # Apply the function to each row in the DataFrame to get the segmented sequences and labels\n    data_df['sequence_chunks'] = None\n    data_df['label_chunks'] = None\n    for idx, row in data_df.iterrows():\n        data_df.at[idx, 'sequence_chunks'], data_df.at[idx, 'label_chunks'] = segment_into_chunks(row)\n\n    # Apply the function with a progress bar using tqdm.pandas\n    data_df['predictions_chunks'] = data_df.progress_apply(get_chunk_predictions, axis=1)\n\n    # Flatten the lists of labels and predictions to calculate metrics\n    true_labels_flat = [label for sublist in data_df['label_chunks'].tolist() for subsublist in sublist for label in subsublist]\n    predictions_flat = [label for sublist in data_df['predictions_chunks'].tolist() for subsublist in sublist for label in subsublist]\n\n    # Calculate the metrics\n    accuracy = accuracy_score(true_labels_flat, predictions_flat)\n    precision = precision_score(true_labels_flat, predictions_flat)\n    recall = recall_score(true_labels_flat, predictions_flat)\n    f1 = f1_score(true_labels_flat, predictions_flat)\n    auc = roc_auc_score(true_labels_flat, predictions_flat)\n    mcc = matthews_corrcoef(true_labels_flat, predictions_flat)\n\n    # Print the metrics\n    print(f'Dataset: {os.path.basename(dataset_path)}')\n    print(f'Accuracy: {accuracy:.4f}')\n    print(f'Precision: {precision:.4f}')\n    print(f'Recall: {recall:.4f}')\n    print(f'F1 Score: {f1:.4f}')\n    print(f'AUC: {auc:.4f}')\n    print(f'MCC: {mcc:.4f}')\n\n    # Collect the metrics\n    accuracy_list.append(accuracy)\n    precision_list.append(precision)\n    recall_list.append(recall)\n    f1_list.append(f1)\n    auc_list.append(auc)\n    mcc_list.append(mcc)\n    print('-' * 50)\n    \n# Calculate the averages\naverage_accuracy = sum(accuracy_list) / len(accuracy_list)\naverage_precision = sum(precision_list) / len(precision_list)\naverage_recall = sum(recall_list) / len(recall_list)\naverage_f1 = sum(f1_list) / len(f1_list)\naverage_auc = sum(auc_list) / len(auc_list)\naverage_mcc = sum(mcc_list) / len(mcc_list)\n\n# Print the averages\nprint(f'Average Accuracy: {average_accuracy:.4f}')\nprint(f'Average Precision: {average_precision:.4f}')\nprint(f'Average Recall: {average_recall:.4f}')\nprint(f'Average F1 Score: {average_f1:.4f}')\nprint(f'Average AUC: {average_auc:.4f}')\nprint(f'Average MCC: {average_mcc:.4f}')\n","metadata":{"execution":{"iopub.status.busy":"2023-09-25T02:35:45.427183Z","iopub.execute_input":"2023-09-25T02:35:45.427688Z","iopub.status.idle":"2023-09-25T02:36:30.423728Z","shell.execute_reply.started":"2023-09-25T02:35:45.427651Z","shell.execute_reply":"2023-09-25T02:36:30.422521Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stderr","text":"Some weights of EsmForTokenClassification were not initialized from the model checkpoint at facebook/esm2_t12_35M_UR50D and are newly initialized: ['classifier.bias', 'classifier.weight']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\nProcessing rows: 100%|██████████| 50/50 [00:02<00:00, 21.48it/s]\n","output_type":"stream"},{"name":"stdout","text":"Dataset: ATP_Validation.txt\nAccuracy: 0.9208\nPrecision: 0.1794\nRecall: 0.3122\nF1 Score: 0.2279\nAUC: 0.6283\nMCC: 0.1974\n--------------------------------------------------\n","output_type":"stream"},{"name":"stderr","text":"Processing rows: 100%|██████████| 27/27 [00:01<00:00, 21.76it/s]\n","output_type":"stream"},{"name":"stdout","text":"Dataset: HEME_Validation.txt\nAccuracy: 0.8935\nPrecision: 0.1195\nRecall: 0.1086\nF1 Score: 0.1138\nAUC: 0.5274\nMCC: 0.0574\n--------------------------------------------------\n","output_type":"stream"},{"name":"stderr","text":"Processing rows: 100%|██████████| 33/33 [00:01<00:00, 22.08it/s]\n","output_type":"stream"},{"name":"stdout","text":"Dataset: AMP_Validation.txt\nAccuracy: 0.9102\nPrecision: 0.1649\nRecall: 0.3597\nF1 Score: 0.2261\nAUC: 0.6454\nMCC: 0.2014\n--------------------------------------------------\n","output_type":"stream"},{"name":"stderr","text":"Processing rows: 100%|██████████| 176/176 [00:08<00:00, 21.49it/s]\n","output_type":"stream"},{"name":"stdout","text":"Dataset: ZN_Validation.txt\nAccuracy: 0.9279\nPrecision: 0.0464\nRecall: 0.1895\nF1 Score: 0.0745\nAUC: 0.5644\nMCC: 0.0653\n--------------------------------------------------\n","output_type":"stream"},{"name":"stderr","text":"Processing rows: 100%|██████████| 14/14 [00:00<00:00, 22.08it/s]\n","output_type":"stream"},{"name":"stdout","text":"Dataset: GDP_Validation.txt\nAccuracy: 0.9259\nPrecision: 0.2930\nRecall: 0.4742\nF1 Score: 0.3622\nAUC: 0.7106\nMCC: 0.3359\n--------------------------------------------------\n","output_type":"stream"},{"name":"stderr","text":"Processing rows: 100%|██████████| 26/26 [00:01<00:00, 22.03it/s]\n","output_type":"stream"},{"name":"stdout","text":"Dataset: FE_Validation.txt\nAccuracy: 0.9415\nPrecision: 0.0733\nRecall: 0.3000\nF1 Score: 0.1178\nAUC: 0.6250\nMCC: 0.1262\n--------------------------------------------------\n","output_type":"stream"},{"name":"stderr","text":"Processing rows: 100%|██████████| 47/47 [00:02<00:00, 20.10it/s]\n","output_type":"stream"},{"name":"stdout","text":"Dataset: ADP_Validation.txt\nAccuracy: 0.9364\nPrecision: 0.2134\nRecall: 0.3528\nF1 Score: 0.2659\nAUC: 0.6544\nMCC: 0.2429\n--------------------------------------------------\n","output_type":"stream"},{"name":"stderr","text":"Processing rows: 100%|██████████| 58/58 [00:02<00:00, 22.06it/s]\n","output_type":"stream"},{"name":"stdout","text":"Dataset: MN_Validation.txt\nAccuracy: 0.9238\nPrecision: 0.0501\nRecall: 0.2616\nF1 Score: 0.0841\nAUC: 0.5972\nMCC: 0.0876\n--------------------------------------------------\n","output_type":"stream"},{"name":"stderr","text":"Processing rows: 100%|██████████| 7/7 [00:00<00:00, 22.15it/s]\n","output_type":"stream"},{"name":"stdout","text":"Dataset: GTP_Validation.txt\nAccuracy: 0.8932\nPrecision: 0.2000\nRecall: 0.4494\nF1 Score: 0.2768\nAUC: 0.6819\nMCC: 0.2502\n--------------------------------------------------\n","output_type":"stream"},{"name":"stderr","text":"Processing rows: 100%|██████████| 217/217 [00:10<00:00, 21.50it/s]\n","output_type":"stream"},{"name":"stdout","text":"Dataset: MG_Validation.txt\nAccuracy: 0.9315\nPrecision: 0.0377\nRecall: 0.1984\nF1 Score: 0.0634\nAUC: 0.5693\nMCC: 0.0620\n--------------------------------------------------\n","output_type":"stream"},{"name":"stderr","text":"Processing rows: 100%|██████████| 52/52 [00:02<00:00, 20.72it/s]\n","output_type":"stream"},{"name":"stdout","text":"Dataset: DNA_Validation.txt\nAccuracy: 0.9105\nPrecision: 0.1549\nRecall: 0.1305\nF1 Score: 0.1417\nAUC: 0.5439\nMCC: 0.0952\n--------------------------------------------------\n","output_type":"stream"},{"name":"stderr","text":"Processing rows: 100%|██████████| 165/165 [00:07<00:00, 21.57it/s]\n","output_type":"stream"},{"name":"stdout","text":"Dataset: CA_Validation.txt\nAccuracy: 0.9237\nPrecision: 0.0352\nRecall: 0.1631\nF1 Score: 0.0579\nAUC: 0.5489\nMCC: 0.0467\n--------------------------------------------------\nAverage Accuracy: 0.9199\nAverage Precision: 0.1307\nAverage Recall: 0.2750\nAverage F1 Score: 0.1677\nAverage AUC: 0.6081\nAverage MCC: 0.1474\n","output_type":"stream"}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}