test

.ipynb_checkpoints/app-checkpoint.py

@@ -1,4 +1,133 @@
+
 import streamlit as st
+import subprocess
+
+subprocesses.run(['git', 'clone', 'https://huggingface.co/ttmn/SolLlama-mtr'])
+
+import sys
+import os
+import torch
+import numpy as np
+import pandas as pd
+import warnings
+import lightning as L
+torch.set_float32_matmul_precision('high')
+warnings.filterwarnings("ignore", module="pl_bolts")
+
+sys.path.append( '../')
+
+import tokenizer_sl, datamodule_finetune_sl, model_finetune_sl, chemllama_mtr, utils_sl
+import auto_evaluator_sl
+
+torch.manual_seed(1004)
+np.random.seed(1004)
+
+smiles_str = st.text_area('Enter SMILE string')
+
+###
+solute_or_solvent = 'solute'
+solute_or_solvent = st.selectbox('Solute or Solvent', ['Solute,' 'Solvent'])
+ver_ft = 0 # version control for FT model & evaluation data # Or it will overwrite the models and results
+batch_size_pair = [64, 64] if solute_or_solvent == 'solute' else [10, 10] # [train, valid(test)] 
+# since 'solute' has very small dataset. So I thinl 10 for train and 10 for valid(test) should be the maximum values.
+lr = 0.0001 
+epochs = 7
+use_freeze = False  # Freeze the model or not # False measn not freezing
+overwrite_level_2 = True
+###
+max_seq_length = 512
+tokenizer = tokenizer_sl.fn_load_tokenizer_llama(
+    max_seq_length=max_seq_length,
+)
+max_length = max_seq_length
+num_workers = 2
+
+# I just reused our previous research code with some modifications.
+dir_main = "./"
+name_model_mtr = "ChemLlama_Medium_30m_vloss_val_loss=0.029_ep_epoch=04.ckpt" 
+
+dir_model_mtr = f"{dir_main}/SolLmama-mtr/{name_model_mtr}"
+
+max_seq_length = 512
+
+tokenizer = tokenizer_sl.fn_load_tokenizer_llama(
+    max_seq_length=max_seq_length,
+)
+max_length = max_seq_length
+num_workers = 2
+
+## FT
+
+ver_ft = 0
+dir_model_ft_to_save = f"{dir_main}/SolLlama-mtr"
+# name_model_ft = 'Solvent.pt'
+name_model_ft = f"{solute_or_solvent}.pt"
+
+# Load dataset for finetune
+batch_size_for_train = batch_size_pair[0]
+batch_size_for_valid = batch_size_pair[1]
+
+data_module = datamodule_finetune_sol.CustomFinetuneDataModule(
+    solute_or_solvent=solute_or_solvent,
+    tokenizer=tokenizer,
+    max_seq_length=max_length,
+    batch_size_train=batch_size_for_train,
+    batch_size_valid=batch_size_for_valid,
+    # num_device=int(config.NUM_DEVICE) * config.NUM_WORKERS_MULTIPLIER,
+    num_device=num_workers,
+)
+
+data_module.prepare_data(smiles_str=smiles_str)
+data_module.setup()
+steps_per_epoch = len(data_module.train_dataloader())
+
+# Load model and optimizer for finetune
+learning_rate = lr
+
+
+model_mtr = chemllama_mtr.ChemLlama.load_from_checkpoint(dir_model_mtr)
+
+
+model_ft = model_finetune_sl.CustomFinetuneModel(
+    model_mtr=model_mtr,
+    steps_per_epoch=steps_per_epoch,
+    warmup_epochs=1,
+    max_epochs=epochs,
+    learning_rate=learning_rate,
+    # dataset_dict=dataset_dict,
+    use_freeze=use_freeze,
+)
+
+# 'SolLlama_solute_vloss_val_loss=0.082_ep_epoch=06.pt'
+
+trainer = L.Trainer(
+    default_root_dir=dir_model_ft_to_save,
+    # profiler=profiler,
+    # logger=csv_logger,
+    accelerator='auto',
+    devices='auto',
+    # accelerator='gpu',
+    # devices=[0],
+    min_epochs=1,
+    max_epochs=epochs,
+    precision=32,
+    # callbacks=[checkpoint_callback]
+)
+
+
+# Predict
+local_model_ft = utils_sl.load_model_ft_with_epoch(
+    class_model_ft=model_ft, 
+    target_epoch=ep,
+    dir_model_ft=dir_model_ft_to_save,
+    name_model_ft=name_model_ft
+)
+
+result = trainer.predict(local_model_ft, data_module)
+result_pred = list()
+result_label = list()
+for bat in range(len(result)):
+    result_pred.append(result[bat][0].squeeze())
+    result_label.append(result[bat][1])
 
-x = st.slider('Slect a value')
-st.write(x, 'squared is :', x * x)
+st.write(result_pred)

.ipynb_checkpoints/auto_evaluator_sl-checkpoint.py

@@ -0,0 +1,297 @@
+import sys
+import os
+import re
+import pandas as pd
+import numpy as np
+
+import lightning as L
+from lightning.pytorch.loggers import CSVLogger
+from lightning.pytorch.profilers import PyTorchProfiler
+from lightning.pytorch.callbacks import ModelCheckpoint
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+# Now you can import from models_mtr
+# from chemllama_mtr import ChemLlama
+import chemllama_mtr
+
+# from .datamodule_finetune import CustomFinetuneDataModule
+import datamodule_finetune_sl
+# from .model_finetune import CustomFinetuneModel
+import model_finetune_sl
+import utils_sol
+
+def auto_evaluator_level_2_sol(
+    dir_model_mtr,
+    # dir_model_mtr_ep_to_save:str,
+    dir_model_ft_to_save:str,
+    tokenizer,
+    max_length:int,
+    # molnet_dict:dict,
+    # list_dataset_to_finetune:list,
+    solute_or_solvent:str,
+    num_workers:int,
+    batch_size_pair=[32, 48],
+    lr=0.0001,
+    overwrite_level_2:bool=False,
+    epochs:int=7,
+    use_freeze:bool=True
+):
+
+    """
+    Evaluate the "one" pretrained MTR model through multiple finetuning benchmarking dataset.
+
+    Parameters:
+    # - dir_model_mtr_ep_to_save (str): The pretrained model for MTR with epoch.
+    #                                    EX with 0 epoch:
+    #                                    /master_dicrectory/pre_trained_model_MTR_name/model_MTR_with_epoch
+    - batch_size_pair: The pair of the train and valid(+test) batch size (e.g. [32, 48] which is [32, int(32*1.5)])
+    - overwrite_level_2 (bool): If there exists such folder that has the same "dir_model_mtr_ep_to_save", overwite it.
+                                Warning! This option is only for "dir_model_mtr_ep_to_save". It's sub directory and files will be overwritten!
+    """
+    
+    
+    assert not (os.path.exists(dir_model_ft_to_save) and overwrite_level_2 == False), f"You sat 'overwrite_level_2' False and '{dir_model_ft_to_save}' already exists. Check it again."
+
+
+    model_mtr = chemllama_mtr.ChemLlama.load_from_checkpoint(dir_model_mtr)
+        
+    # # local_dataset_to_finetune is a key of molnet_dict
+    # list_local_finetuned_result = list()
+    # for local_dataset_to_finetune in list_dataset_to_finetune:
+        
+    # dataset_dict = molnet_dict[local_dataset_to_finetune]
+    # dataset_dict["dataset_name"] = local_dataset_to_finetune
+    
+    # dir_model_ft = f"{dir_model_mtr_ep_to_save}/{dataset_dict['dataset_name']}"
+    dir_model_ft = f"{dir_model_ft_to_save}"
+    # name_model_ft = utils_sol.model_ft_namer(dataset_dict['dataset_name'])
+    name_model_ft = f"SolLlama_{solute_or_solvent}"
+
+    # array_level_1, model_ft, data_loader_test
+    array_level_1 = auto_evaluator_level_1_sol(
+        model_mtr=model_mtr, 
+        dir_model_ft=dir_model_ft, 
+        name_model_ft=name_model_ft, 
+        # dataset_dict=dataset_dict, 
+        solute_or_solvent=solute_or_solvent,
+        tokenizer=tokenizer, 
+        max_length=max_length,
+        num_workers=num_workers,
+        batch_size_pair=batch_size_pair,
+        lr=lr,
+        epochs=epochs, 
+        use_freeze=use_freeze,
+    )
+    
+    return array_level_1
+    
+        # list_local_finetuned_result.append(array_level_1)
+        
+    # array_level_2 = np.vstack(list_local_finetuned_result)
+    # array_level_2 shaped (number of epochs x len(list_dataset_to_finetune), number of columns at the bottom)
+    # dataset_name, task, RMSE, MAE, p_value mantissam, p_value exponent, epoch, loss, loss_ranking, metric_1_ranking
+    
+    # return array_level_2
+
+def auto_evaluator_level_1_sol(
+    model_mtr, 
+    dir_model_ft:str, 
+    name_model_ft:str, 
+    # dataset_dict:dict, 
+    solute_or_solvent:str,
+    tokenizer, 
+    max_length:int,
+    num_workers:int, ##
+    batch_size_pair=[32, 48],
+    lr=0.0001,
+    epochs:int=7,
+    use_freeze:bool=True,
+):
+
+    """
+    Automate the entire process including preparing "one" finetuning dataset + finetuing + evalulation.
+    This is a step before the level 2 evaluate automation.
+
+    Parameters:
+    - model_mtr: The pretrained model for MTR.
+    - dir_model_ft (str): The directory where the model to be stored.
+    - name_model_ft (str): The name of the model for finetune to be titled.
+                           An example of the directory of the fintuned model with 0 epoch:
+                           {dir_folder}/{name_model_ft}_ep_000
+    - batch_size_pair: The pair of the train and valid(+test) batch size (e.g. [32, 48] which is [32, int(32*1.5)])
+    """
+    
+    csv_logger = CSVLogger(
+        save_dir=dir_model_ft,
+        name=name_model_ft,
+        version=0,
+    )
+
+    checkpoint_callback = ModelCheckpoint(
+        monitor='val_loss',
+        filename=name_model_ft + '_vloss_{val_loss:.3f}_ep_{epoch:02d}',
+        every_n_epochs=1,
+        save_top_k=-1,
+        enable_version_counter=False, # keep the version == 0
+        save_weights_only=True,
+    )
+    checkpoint_callback.FILE_EXTENSION = ".pt"
+    
+    # Load dataset for finetune
+    batch_size_for_train = batch_size_pair[0]
+    batch_size_for_valid = batch_size_pair[1]
+
+    data_module = datamodule_finetune_sol.CustomFinetuneDataModule(
+        solute_or_solvent=solute_or_solvent,
+        tokenizer=tokenizer,
+        max_seq_length=max_length,
+        batch_size_train=batch_size_for_train,
+        batch_size_valid=batch_size_for_valid,
+        # num_device=int(config.NUM_DEVICE) * config.NUM_WORKERS_MULTIPLIER,
+        num_device=num_workers,
+    )
+    data_module.prepare_data()
+    data_module.setup()
+    steps_per_epoch = len(data_module.train_dataloader())
+
+    # Load model and optimizer for finetune
+    learning_rate = lr
+        
+    model_ft = model_finetune_sol.CustomFinetuneModel(
+        model_mtr=model_mtr,
+        steps_per_epoch=steps_per_epoch,
+        warmup_epochs=1,
+        max_epochs=epochs,
+        learning_rate=learning_rate,
+        # dataset_dict=dataset_dict,
+        use_freeze=use_freeze,
+    )
+    
+    trainer = L.Trainer(
+        default_root_dir=dir_model_ft,
+        # profiler=profiler,
+        logger=csv_logger,
+        accelerator='auto',
+        devices='auto',
+        # accelerator='gpu',
+        # devices=[0],
+        min_epochs=1,
+        max_epochs=epochs,
+        precision=32,
+        callbacks=[checkpoint_callback]
+    )
+    trainer.fit(model_ft, data_module)
+    trainer.validate(model_ft, data_module)
+
+    list_validation_loss = pd.read_csv(f"{dir_model_ft}/{name_model_ft}/version_0/metrics.csv", usecols=['val_loss'])['val_loss'].dropna().tolist()[:epochs]
+
+    # class_model_ft = CustomFinetuneModel
+    # Level 1 Automation - Evaulate the finetuned model through every epoch
+    array_level_1 = auto_evaluator_level_1_sub_sol(
+        class_model_ft=model_ft, 
+        list_validation_loss=list_validation_loss, 
+        dir_model_ft=dir_model_ft, 
+        name_model_ft=name_model_ft, 
+        data_module=data_module,
+        # dataset_dict=dataset_dict,
+        solute_or_solvent=solute_or_solvent,
+        trainer=trainer
+    )
+    
+    return array_level_1
+
+def auto_evaluator_level_1_sub_sol(
+    class_model_ft,
+    list_validation_loss,
+    dir_model_ft:str,
+    name_model_ft:str,
+    data_module,
+    # dataset_dict:dict,
+    solute_or_solvent:str,
+    trainer,
+):
+
+    """
+    Evaluate the finetuned model by a single finetuning dataset.
+
+    Guides for some parameters:
+    - model_mtr: The pretrained model for MTR.
+    - dir_model_ft (str): The directory where the model to be stored.
+    - name_model_ft (str): The name of the model for finetune to be titled.
+                           An example of the directory of the fintuned model with 0 epoch:
+                           {dir_folder}/{name_model_ft}_ep_000
+    """
+    
+    array_loss_ranking = utils_sol.rank_value_sol(
+        list_value=list_validation_loss, 
+        # dataset_dict=dataset_dict,
+        is_loss=True,
+    )
+    # ranking : lower the better. ranking starting from 0
+
+    print("- Epoch starts from 0")
+    print("=======================================")
+    
+    list_level_1 = list()
+    for ep in range(len(list_validation_loss)):
+
+        local_model_ft = utils_sol.load_model_ft_with_epoch(
+            class_model_ft=class_model_ft, 
+            target_epoch=ep,
+            dir_model_ft=dir_model_ft,
+            name_model_ft=name_model_ft
+        )
+        
+        result = trainer.predict(local_model_ft, data_module)
+        result_pred = list()
+        result_label = list()
+        for bat in range(len(result)):
+            result_pred.append(result[bat][0].squeeze())
+            result_label.append(result[bat][1])
+
+        list_local_model_ft_result = utils_sol.model_evalulator_sol(
+            array_predictions=np.vstack(result_pred),
+            array_labels=np.vstack(result_label),
+            # dataset_dict=dataset_dict, 
+            solute_or_solvent=solute_or_solvent,
+            show_plot=False,
+            print_result=False,
+        )
+        # dataset_name, task, RMSE, MAE, p_value mantissam, p_value exponent
+        
+        # add epoch (starting from 0) to the right 
+        list_local_model_ft_result.append(ep)
+        # dataset_name, task, metric1 (RMSE or ROC-AUC), metric2 (MAE or None), p_value mantissam, p_value exponent, epoch
+        
+        list_level_1.append(list_local_model_ft_result)
+    print("=======================================")
+    print("=======================================")
+
+    # to get the metric_1 ranking
+    array_level_1 = np.array(list_level_1)
+    array_metric_1 = array_level_1[:, 2].astype('float32')
+    array_metric_1_ranking = utils_sol.rank_value_sol(list_value=array_metric_1,
+                                              # dataset_dict=dataset_dict,
+                                              is_loss=False)
+
+    # add loss, and ranking of the loss value to the right
+    # reg: lower the better, class: higher the better
+    array_level_1 = np.hstack((list_level_1, 
+                               np.expand_dims(list_validation_loss, axis=1), 
+                               np.expand_dims(array_loss_ranking, axis=1),
+                               np.expand_dims(array_metric_1_ranking, axis=1))) 
+    # solute_or_solvent, RMSE, MAE, p_value mantissam, p_value exponent, epoch, loss, loss_ranking, metric_1_ranking
+    
+    return array_level_1 
+    #################################### EX #########################################
+    # list_column_names = ['solute_or_solvent', 
+    #                      'metric_1', 
+    #                      'metric_2', 
+    #                      'p_value_mantissa', 
+    #                      'p_value_exponent', 
+    #                      'epoch', 
+    #                      'loss',
+    #                      'loss_ranking',
+    #                      'metric_1_ranking']
+    # df_evaluation_level_1 = pd.DataFrame(array_level_1, columns=list_column_names)
+    #################################################################################

.ipynb_checkpoints/datamodule_finetune_sl-checkpoint.py

@@ -0,0 +1,120 @@
+import lightning as L
+import torch
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import train_test_split
+
+from torch.utils.data import Dataset, DataLoader
+from transformers import DataCollatorWithPadding
+
+
+class CustomLlamaDatasetAbraham(Dataset):
+    def __init__(self, df, tokenizer, max_seq_length):
+        self.keys = df.iloc[:, 0] # 1D array
+        self.labels = df.iloc[:, 1:] # 2D array
+        self.tokenizer = tokenizer
+        self.max_seq_length = max_seq_length
+
+    def __len__(self):
+        return self.keys.shape[0]
+
+    def fn_token_encode(self, smiles):
+        return self.tokenizer(
+            smiles,
+            truncation=True,
+            padding="max_length",
+            max_length=self.max_seq_length,
+        )
+
+    def __getitem__(self, idx):
+        local_encoded = self.fn_token_encode(self.keys.iloc[idx])
+        
+        return {
+            "input_ids": torch.tensor(local_encoded["input_ids"]),
+            "attention_mask": torch.tensor(local_encoded["attention_mask"]),
+            "labels": torch.tensor(self.labels.iloc[idx]),
+        }
+
+
+class CustomFinetuneDataModule(L.LightningDataModule):
+    def __init__(
+        self,
+        solute_or_solvent,
+        tokenizer,
+        max_seq_length,
+        batch_size_train,
+        batch_size_valid,
+        num_device,
+    ):
+        super().__init__()
+
+        self.solute_or_solvent = solute_or_solvent
+        self.tokenizer = tokenizer
+        self.max_seq_length = max_seq_length
+        self.batch_size_train = batch_size_train
+        self.batch_size_valid = batch_size_valid
+        self.data_collator = DataCollatorWithPadding(self.tokenizer)
+        self.num_device = num_device
+    
+
+    def prepare_data(self, smiles_str:str):
+        # self.list_df = load_abraham(self.solute_or_solvent)
+        self.smiles_str = smiles_str
+
+    def setup(self, stage=None):
+        # self.train_df, self.valid_df, self.test_df = self.list_df
+        self.train_df = None
+        self.valid_df = None
+        self.test_df = self.smiles_str
+
+    def train_dataloader(self):
+        return DataLoader(
+            dataset=CustomLlamaDatasetAbraham(
+                self.train_df, self.tokenizer, self.max_seq_length,
+            ),
+            batch_size=self.batch_size_train,
+            num_workers=self.num_device,
+            collate_fn=self.data_collator,
+            shuffle=True,
+        )
+
+    def val_dataloader(self):
+        return DataLoader(
+            dataset=CustomLlamaDatasetAbraham(
+                self.valid_df, self.tokenizer, self.max_seq_length,
+            ),
+            batch_size=self.batch_size_valid,
+            num_workers=self.num_device,
+            collate_fn=self.data_collator,
+            shuffle=False,
+        )
+
+    def test_dataloader(self):
+        return DataLoader(
+            dataset=CustomLlamaDatasetAbraham(
+                self.test_df, self.tokenizer, self.max_seq_length,
+            ),
+            batch_size=self.batch_size_valid,
+            num_workers=self.num_device,
+            collate_fn=self.data_collator,
+            shuffle=False,
+        )
+        
+    # It uses test_df
+    def predict_dataloader(self): 
+        return DataLoader(
+            dataset=CustomLlamaDatasetAbraham(
+                self.test_df, self.tokenizer, self.max_seq_length,
+            ),
+            batch_size=self.batch_size_valid,
+            num_workers=self.num_device,
+            collate_fn=self.data_collator,
+            shuffle=False,
+        )
+
+
+
+
+
+
+        

.ipynb_checkpoints/dict_dtype_slpy-checkpoint

@@ -0,0 +1,9 @@
+dict_dtype = {
+    "solute_or_solvent": "str",
+    "metric_1": "float32",
+    "metric_2": "float32",
+    "epoch": "int32",
+    "loss": "float32",
+    "loss_ranking": "int32",
+    "metric_1_ranking": "int32",
+}

.ipynb_checkpoints/requirements-checkpoint.txt

@@ -0,0 +1,12 @@
+torch
+transformers
+lightning
+lightning-bolts
+numpy
+pytorch-ignite
+pytorch-lightning
+pytorch-lightning-bolts
+pytorch-warmup
+scikit-learn
+scipy
+seaborn

.ipynb_checkpoints/run_auto_llama_cuda0-checkpoint.py

@@ -0,0 +1,122 @@
+import sys
+import os
+
+# This means you will use the first GPU among the four GPUs in our case.
+# "0", "1", "2", "3". Since FT dataset is small, using one GPU should be proper.
+os.environ["CUDA_VISIBLE_DEVICES"]= "0"
+
+
+import torch
+import numpy as np
+import pandas as pd
+import warnings
+import lightning as L
+torch.set_float32_matmul_precision('high')
+
+# Filter out FutureWarning and UnderReviewWarning messages from pl_bolts
+warnings.filterwarnings("ignore", module="pl_bolts")
+
+# Add the parent directory to sys.path
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+import tokenizer_sl
+import auto_evaluator_sl
+
+print(os.path.dirname(__file__))
+
+
+torch.manual_seed(1004)
+np.random.seed(1004)
+
+print(os.getcwd())
+
+"""
+Note 1 to Dr. Lang
+
+I have checked that when we not freeze the MTR model, then the test loss values are keep decreasing when I set the epochs as 7.
+(At least for solute.) So We you may try to run more epochs if you want. But Solvent may be already overfitted or will get soon since it has only few data.
+
+Using learning rate bigger than the default setting is not that recommanded since we don't freeze the MTR model.
+But lower lr could work.
+
+Be aware of doing version control (ver_ft). Make sure you keep the same version for both 'solute' and 'solvent' otherwise, you will get confused.
+
+The variable "dir_model_ft_to_save" is where the FT model get saved.
+The result csv files will be located at 'evaluations/corresponding version/solute and (or) solvent.csv'
+
+You can run this code by
+cd ~/SolLlama
+python run_auto_llama_cuda0.py
+
+But makes sure you are running this in your virtual environment that all requirements_cuda118.txt installed
+"""
+
+
+"""
+# You can run both 'solute' and 'solvent' at one run by doing the below
+for solute_or_solvent in ['solute' ,'solvent']:
+    The REST of the codes except the variant solute_or_solvent right below with this (SAME) indentation levels
+"""
+#### Hyper Parameters ##### <- You can control these parameters as you want
+# solute_or_solvent = 'solvent'
+solute_or_solvent = 'solute'
+ver_ft = 0 # version control for FT model & evaluation data # Or it will overwrite the models and results
+batch_size_pair = [64, 64] if solute_or_solvent == 'solute' else [10, 10] # [train, valid(test)] 
+# since 'solute' has very small dataset. So I thinl 10 for train and 10 for valid(test) should be the maximum values.
+lr = 0.0001 
+epochs = 7
+use_freeze = False  # Freeze the model or not # False measn not freezing
+overwrite_level_2 = True # If you don't want to overwrite the models and csv files, then change this to False
+###########################
+
+
+# I just reused our previous research code with some modifications.
+dir_main = "/home/ylee/SolLlama"
+name_model_mtr = "ChemLlama_Medium_30m_vloss_val_loss=0.029_ep_epoch=04.ckpt" 
+
+dir_model_mtr = f"{dir_main}/model_mtr/{name_model_mtr}"
+
+max_seq_length = 512
+
+tokenizer = tokenizer_sol.fn_load_tokenizer_llama(
+    max_seq_length=max_seq_length,
+)
+max_length = max_seq_length
+num_workers = 2
+
+dir_model_ft_to_save = f"{dir_main}/save_models_ft/ft_version_{ver_ft}"
+
+array_level_2 = auto_evaluator_sol.auto_evaluator_level_2_sol(
+    dir_model_mtr=dir_model_mtr,
+    dir_model_ft_to_save=dir_model_ft_to_save,
+    tokenizer=tokenizer,
+    max_length=max_seq_length,
+    solute_or_solvent=solute_or_solvent,
+    num_workers=num_workers,
+    batch_size_pair=batch_size_pair,
+    lr=lr,
+    overwrite_level_2=overwrite_level_2,
+    epochs=epochs,
+    use_freeze=use_freeze,
+)
+
+print(array_level_2.shape)
+print(array_level_2)
+
+list_column_names_level_2 = [
+    'solute_or_solvent', 
+    'metric_1', 
+    'metric_2', 
+    'epoch', 
+    'loss',
+    'loss_ranking',
+    'metric_1_ranking'
+]
+
+df_evaluation_level_2 = pd.DataFrame(array_level_2, columns=list_column_names_level_2)
+
+os.makedirs(f'{os.path.dirname(__file__)}/evaluations/ft_version_{ver_ft}', exist_ok=True)
+df_evaluation_level_2.to_csv(f'{os.path.dirname(__file__)}/evaluations/ft_version_{ver_ft}/{solute_or_solvent}.csv', index=False)
+
+
+

.ipynb_checkpoints/tokenizer_sl-checkpoint.py

@@ -0,0 +1,35 @@
+from transformers import LlamaTokenizerFast
+import json
+import os
+
+
+def fn_load_tokenizer_llama(
+    max_seq_length,
+    dir_tokenizer: str = "./tokenizer.json",
+    # dir_tokenizer:str = os.path.abspath(os.path.join(os.getcwd(), '..', "models_mtr/tokenizer.json")), # for JUP
+    add_eos_token:bool = True,
+):
+
+    tokenizer = LlamaTokenizerFast(
+        tokenizer_file=dir_tokenizer,
+        model_max_length=max_seq_length,
+        padding_side="right",
+        bos_token="<s>",
+        eos_token="</s>",
+        unk_token="<unk>",
+        add_eos_token=add_eos_token,
+    )
+    tokenizer.add_special_tokens({"pad_token": "<pad>", "sep_token": "</s>", "cls_token": "<s>", "mask_token":"<mask>"})
+    # tokenizer.add_special_tokens({"pad_token": "<pad>"})
+
+    return tokenizer
+
+def fn_load_descriptor_list(
+    key_descriptor_list,
+    dir_descriptor_list,
+):
+
+    with open(dir_descriptor_list, "r") as js:
+        list_descriptor = json.load(js)[key_descriptor_list]
+
+    return list_descriptor

.ipynb_checkpoints/utils_sl-checkpoint.py

@@ -0,0 +1,114 @@
+from sklearn.metrics import roc_curve, auc, roc_auc_score
+from sklearn.metrics import mean_squared_error
+from sklearn.metrics import r2_score
+from sklearn.metrics import mean_absolute_error
+from scipy.stats import spearmanr
+import matplotlib.pyplot as plt
+
+import numpy as np
+import os
+
+def model_evalulator_sol(
+    array_predictions, 
+    array_labels, 
+    # dataset_dict:dict, 
+    solute_or_solvent:str,
+    show_plot:bool=True, 
+    print_result:bool=True,
+):
+    
+    if print_result:        
+        print(f"Dataset : {solute_or_solvent}")
+        print("N:", array_labels.shape[0])
+    
+    fig, ax = plt.subplots()
+
+    metric = mean_squared_error(array_labels, array_predictions, squared=False) #RMSE
+    r2 = r2_score(array_labels, array_predictions)
+    metric2 = mean_absolute_error(array_labels, array_predictions) # MAE
+    ax.scatter(array_labels, array_predictions)
+    ax.set_title("Scatter Plot of Labels vs Predictions")
+    ax.set_xlabel("Labels")
+    ax.set_ylabel("Predictions")
+
+    if print_result:
+        print("R2:", r2)
+        print("Root Mean Square Error:", metric)
+        print("Mean Absolute Error:", metric2)
+
+    # correlation, p_value = spearmanr(array_labels, array_predictions)
+
+    # if print_result:
+    #     print("Spearman correlation:", correlation)
+    #     print("p-value:", p_value)
+    #     print("=======================================")
+    
+    xmin, xmax = ax.get_xlim()
+    ax.set_ylim(xmin, xmax)
+    
+    if not show_plot:
+        plt.ioff()
+        plt.clf()
+        plt.close()
+    else :
+        plt.show()
+    
+    # metrict 1 - ROC score (classification) | RMSE (regression)
+    # metric 2 - None (classification) | MAE ( regression)
+    round_decimal = 6
+    if metric2 != None:
+        metric2 = round(metric2, round_decimal)
+
+    # list_p_value = str(p_value).split('e')
+    # p_value_mantissa = round(float(list_p_value[0]), round_decimal)
+    # if len(list_p_value) == 2:
+    #     p_value_exponent = int(list_p_value[1])
+    # else:
+    #     p_value_exponent = None
+
+    return [solute_or_solvent,
+            round(metric, round_decimal), 
+            metric2]
+    # return [solute_or_solvent,
+    #         round(metric, round_decimal), 
+    #         metric2,
+    #         p_value_mantissa,
+    #         p_value_exponent]
+
+# from .model_finetune import CustomFinetuneModel
+# import model_finetune_sol
+import torch
+def load_model_ft_with_epoch(class_model_ft,
+                             target_epoch:int,
+                             dir_model_ft:str,
+                             name_model_ft:str):
+    # dir_model_ft level 1
+    # ex /main/model_mtr/model_mtr_ep/dataset
+    
+    dir_all_model_ft = f"{dir_model_ft}/{name_model_ft}/version_0/checkpoints/"
+    list_files_in_dir_model_ft = os.listdir(dir_all_model_ft)
+    # extension = '.ckpt'
+    extension = '.pt'
+    list_model_ft_in_the_dir = sorted(list_files_in_dir_model_ft, key=lambda x: float(x.split('=')[-1].split('.')[0]))
+    
+    print(f"Loaded model with epoch {target_epoch}")
+    dir_target_model_ft = f"{dir_all_model_ft}/{list_model_ft_in_the_dir[target_epoch]}"
+    
+    # class_model_ft.load_from_checkpoint(dir_target_model_ft)
+    
+    loaded_state_dict = torch.load(dir_target_model_ft)
+    class_model_ft.load_state_dict(loaded_state_dict['state_dict'])
+    
+    return class_model_ft # now is model_ft
+
+from scipy.stats import rankdata
+# rankdata does not consider decimal places!
+def rank_value_sol(
+    list_value, 
+    # dataset_dict:dict, 
+    is_loss:bool=True,
+):
+    
+    list_value = np.array(list_value)
+
+    return np.array(rankdata(list_value * 100000, method='min')) - 1

app.py

@@ -1,4 +1,133 @@
+
 import streamlit as st
+import subprocess
+
+subprocesses.run(['git', 'clone', 'https://huggingface.co/ttmn/SolLlama-mtr'])
+
+import sys
+import os
+import torch
+import numpy as np
+import pandas as pd
+import warnings
+import lightning as L
+torch.set_float32_matmul_precision('high')
+warnings.filterwarnings("ignore", module="pl_bolts")
+
+sys.path.append( '../')
+
+import tokenizer_sl, datamodule_finetune_sl, model_finetune_sl, chemllama_mtr, utils_sl
+import auto_evaluator_sl
+
+torch.manual_seed(1004)
+np.random.seed(1004)
+
+smiles_str = st.text_area('Enter SMILE string')
+
+###
+solute_or_solvent = 'solute'
+solute_or_solvent = st.selectbox('Solute or Solvent', ['Solute,' 'Solvent'])
+ver_ft = 0 # version control for FT model & evaluation data # Or it will overwrite the models and results
+batch_size_pair = [64, 64] if solute_or_solvent == 'solute' else [10, 10] # [train, valid(test)] 
+# since 'solute' has very small dataset. So I thinl 10 for train and 10 for valid(test) should be the maximum values.
+lr = 0.0001 
+epochs = 7
+use_freeze = False  # Freeze the model or not # False measn not freezing
+overwrite_level_2 = True
+###
+max_seq_length = 512
+tokenizer = tokenizer_sl.fn_load_tokenizer_llama(
+    max_seq_length=max_seq_length,
+)
+max_length = max_seq_length
+num_workers = 2
+
+# I just reused our previous research code with some modifications.
+dir_main = "./"
+name_model_mtr = "ChemLlama_Medium_30m_vloss_val_loss=0.029_ep_epoch=04.ckpt" 
+
+dir_model_mtr = f"{dir_main}/SolLmama-mtr/{name_model_mtr}"
+
+max_seq_length = 512
+
+tokenizer = tokenizer_sl.fn_load_tokenizer_llama(
+    max_seq_length=max_seq_length,
+)
+max_length = max_seq_length
+num_workers = 2
+
+## FT
+
+ver_ft = 0
+dir_model_ft_to_save = f"{dir_main}/SolLlama-mtr"
+# name_model_ft = 'Solvent.pt'
+name_model_ft = f"{solute_or_solvent}.pt"
+
+# Load dataset for finetune
+batch_size_for_train = batch_size_pair[0]
+batch_size_for_valid = batch_size_pair[1]
+
+data_module = datamodule_finetune_sol.CustomFinetuneDataModule(
+    solute_or_solvent=solute_or_solvent,
+    tokenizer=tokenizer,
+    max_seq_length=max_length,
+    batch_size_train=batch_size_for_train,
+    batch_size_valid=batch_size_for_valid,
+    # num_device=int(config.NUM_DEVICE) * config.NUM_WORKERS_MULTIPLIER,
+    num_device=num_workers,
+)
+
+data_module.prepare_data(smiles_str=smiles_str)
+data_module.setup()
+steps_per_epoch = len(data_module.train_dataloader())
+
+# Load model and optimizer for finetune
+learning_rate = lr
+
+
+model_mtr = chemllama_mtr.ChemLlama.load_from_checkpoint(dir_model_mtr)
+
+
+model_ft = model_finetune_sl.CustomFinetuneModel(
+    model_mtr=model_mtr,
+    steps_per_epoch=steps_per_epoch,
+    warmup_epochs=1,
+    max_epochs=epochs,
+    learning_rate=learning_rate,
+    # dataset_dict=dataset_dict,
+    use_freeze=use_freeze,
+)
+
+# 'SolLlama_solute_vloss_val_loss=0.082_ep_epoch=06.pt'
+
+trainer = L.Trainer(
+    default_root_dir=dir_model_ft_to_save,
+    # profiler=profiler,
+    # logger=csv_logger,
+    accelerator='auto',
+    devices='auto',
+    # accelerator='gpu',
+    # devices=[0],
+    min_epochs=1,
+    max_epochs=epochs,
+    precision=32,
+    # callbacks=[checkpoint_callback]
+)
+
+
+# Predict
+local_model_ft = utils_sl.load_model_ft_with_epoch(
+    class_model_ft=model_ft, 
+    target_epoch=ep,
+    dir_model_ft=dir_model_ft_to_save,
+    name_model_ft=name_model_ft
+)
+
+result = trainer.predict(local_model_ft, data_module)
+result_pred = list()
+result_label = list()
+for bat in range(len(result)):
+    result_pred.append(result[bat][0].squeeze())
+    result_label.append(result[bat][1])
 
-x = st.slider('Slect a value')
-st.write(x, 'squared is :', x * x)
+st.write(result_pred)

auto_evaluator_sl.py

@@ -0,0 +1,297 @@
+import sys
+import os
+import re
+import pandas as pd
+import numpy as np
+
+import lightning as L
+from lightning.pytorch.loggers import CSVLogger
+from lightning.pytorch.profilers import PyTorchProfiler
+from lightning.pytorch.callbacks import ModelCheckpoint
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+# Now you can import from models_mtr
+# from chemllama_mtr import ChemLlama
+import chemllama_mtr
+
+# from .datamodule_finetune import CustomFinetuneDataModule
+import datamodule_finetune_sl
+# from .model_finetune import CustomFinetuneModel
+import model_finetune_sl
+import utils_sol
+
+def auto_evaluator_level_2_sol(
+    dir_model_mtr,
+    # dir_model_mtr_ep_to_save:str,
+    dir_model_ft_to_save:str,
+    tokenizer,
+    max_length:int,
+    # molnet_dict:dict,
+    # list_dataset_to_finetune:list,
+    solute_or_solvent:str,
+    num_workers:int,
+    batch_size_pair=[32, 48],
+    lr=0.0001,
+    overwrite_level_2:bool=False,
+    epochs:int=7,
+    use_freeze:bool=True
+):
+
+    """
+    Evaluate the "one" pretrained MTR model through multiple finetuning benchmarking dataset.
+
+    Parameters:
+    # - dir_model_mtr_ep_to_save (str): The pretrained model for MTR with epoch.
+    #                                    EX with 0 epoch:
+    #                                    /master_dicrectory/pre_trained_model_MTR_name/model_MTR_with_epoch
+    - batch_size_pair: The pair of the train and valid(+test) batch size (e.g. [32, 48] which is [32, int(32*1.5)])
+    - overwrite_level_2 (bool): If there exists such folder that has the same "dir_model_mtr_ep_to_save", overwite it.
+                                Warning! This option is only for "dir_model_mtr_ep_to_save". It's sub directory and files will be overwritten!
+    """
+    
+    
+    assert not (os.path.exists(dir_model_ft_to_save) and overwrite_level_2 == False), f"You sat 'overwrite_level_2' False and '{dir_model_ft_to_save}' already exists. Check it again."
+
+
+    model_mtr = chemllama_mtr.ChemLlama.load_from_checkpoint(dir_model_mtr)
+        
+    # # local_dataset_to_finetune is a key of molnet_dict
+    # list_local_finetuned_result = list()
+    # for local_dataset_to_finetune in list_dataset_to_finetune:
+        
+    # dataset_dict = molnet_dict[local_dataset_to_finetune]
+    # dataset_dict["dataset_name"] = local_dataset_to_finetune
+    
+    # dir_model_ft = f"{dir_model_mtr_ep_to_save}/{dataset_dict['dataset_name']}"
+    dir_model_ft = f"{dir_model_ft_to_save}"
+    # name_model_ft = utils_sol.model_ft_namer(dataset_dict['dataset_name'])
+    name_model_ft = f"SolLlama_{solute_or_solvent}"
+
+    # array_level_1, model_ft, data_loader_test
+    array_level_1 = auto_evaluator_level_1_sol(
+        model_mtr=model_mtr, 
+        dir_model_ft=dir_model_ft, 
+        name_model_ft=name_model_ft, 
+        # dataset_dict=dataset_dict, 
+        solute_or_solvent=solute_or_solvent,
+        tokenizer=tokenizer, 
+        max_length=max_length,
+        num_workers=num_workers,
+        batch_size_pair=batch_size_pair,
+        lr=lr,
+        epochs=epochs, 
+        use_freeze=use_freeze,
+    )
+    
+    return array_level_1
+    
+        # list_local_finetuned_result.append(array_level_1)
+        
+    # array_level_2 = np.vstack(list_local_finetuned_result)
+    # array_level_2 shaped (number of epochs x len(list_dataset_to_finetune), number of columns at the bottom)
+    # dataset_name, task, RMSE, MAE, p_value mantissam, p_value exponent, epoch, loss, loss_ranking, metric_1_ranking
+    
+    # return array_level_2
+
+def auto_evaluator_level_1_sol(
+    model_mtr, 
+    dir_model_ft:str, 
+    name_model_ft:str, 
+    # dataset_dict:dict, 
+    solute_or_solvent:str,
+    tokenizer, 
+    max_length:int,
+    num_workers:int, ##
+    batch_size_pair=[32, 48],
+    lr=0.0001,
+    epochs:int=7,
+    use_freeze:bool=True,
+):
+
+    """
+    Automate the entire process including preparing "one" finetuning dataset + finetuing + evalulation.
+    This is a step before the level 2 evaluate automation.
+
+    Parameters:
+    - model_mtr: The pretrained model for MTR.
+    - dir_model_ft (str): The directory where the model to be stored.
+    - name_model_ft (str): The name of the model for finetune to be titled.
+                           An example of the directory of the fintuned model with 0 epoch:
+                           {dir_folder}/{name_model_ft}_ep_000
+    - batch_size_pair: The pair of the train and valid(+test) batch size (e.g. [32, 48] which is [32, int(32*1.5)])
+    """
+    
+    csv_logger = CSVLogger(
+        save_dir=dir_model_ft,
+        name=name_model_ft,
+        version=0,
+    )
+
+    checkpoint_callback = ModelCheckpoint(
+        monitor='val_loss',
+        filename=name_model_ft + '_vloss_{val_loss:.3f}_ep_{epoch:02d}',
+        every_n_epochs=1,
+        save_top_k=-1,
+        enable_version_counter=False, # keep the version == 0
+        save_weights_only=True,
+    )
+    checkpoint_callback.FILE_EXTENSION = ".pt"
+    
+    # Load dataset for finetune
+    batch_size_for_train = batch_size_pair[0]
+    batch_size_for_valid = batch_size_pair[1]
+
+    data_module = datamodule_finetune_sol.CustomFinetuneDataModule(
+        solute_or_solvent=solute_or_solvent,
+        tokenizer=tokenizer,
+        max_seq_length=max_length,
+        batch_size_train=batch_size_for_train,
+        batch_size_valid=batch_size_for_valid,
+        # num_device=int(config.NUM_DEVICE) * config.NUM_WORKERS_MULTIPLIER,
+        num_device=num_workers,
+    )
+    data_module.prepare_data()
+    data_module.setup()
+    steps_per_epoch = len(data_module.train_dataloader())
+
+    # Load model and optimizer for finetune
+    learning_rate = lr
+        
+    model_ft = model_finetune_sol.CustomFinetuneModel(
+        model_mtr=model_mtr,
+        steps_per_epoch=steps_per_epoch,
+        warmup_epochs=1,
+        max_epochs=epochs,
+        learning_rate=learning_rate,
+        # dataset_dict=dataset_dict,
+        use_freeze=use_freeze,
+    )
+    
+    trainer = L.Trainer(
+        default_root_dir=dir_model_ft,
+        # profiler=profiler,
+        logger=csv_logger,
+        accelerator='auto',
+        devices='auto',
+        # accelerator='gpu',
+        # devices=[0],
+        min_epochs=1,
+        max_epochs=epochs,
+        precision=32,
+        callbacks=[checkpoint_callback]
+    )
+    trainer.fit(model_ft, data_module)
+    trainer.validate(model_ft, data_module)
+
+    list_validation_loss = pd.read_csv(f"{dir_model_ft}/{name_model_ft}/version_0/metrics.csv", usecols=['val_loss'])['val_loss'].dropna().tolist()[:epochs]
+
+    # class_model_ft = CustomFinetuneModel
+    # Level 1 Automation - Evaulate the finetuned model through every epoch
+    array_level_1 = auto_evaluator_level_1_sub_sol(
+        class_model_ft=model_ft, 
+        list_validation_loss=list_validation_loss, 
+        dir_model_ft=dir_model_ft, 
+        name_model_ft=name_model_ft, 
+        data_module=data_module,
+        # dataset_dict=dataset_dict,
+        solute_or_solvent=solute_or_solvent,
+        trainer=trainer
+    )
+    
+    return array_level_1
+
+def auto_evaluator_level_1_sub_sol(
+    class_model_ft,
+    list_validation_loss,
+    dir_model_ft:str,
+    name_model_ft:str,
+    data_module,
+    # dataset_dict:dict,
+    solute_or_solvent:str,
+    trainer,
+):
+
+    """
+    Evaluate the finetuned model by a single finetuning dataset.
+
+    Guides for some parameters:
+    - model_mtr: The pretrained model for MTR.
+    - dir_model_ft (str): The directory where the model to be stored.
+    - name_model_ft (str): The name of the model for finetune to be titled.
+                           An example of the directory of the fintuned model with 0 epoch:
+                           {dir_folder}/{name_model_ft}_ep_000
+    """
+    
+    array_loss_ranking = utils_sol.rank_value_sol(
+        list_value=list_validation_loss, 
+        # dataset_dict=dataset_dict,
+        is_loss=True,
+    )
+    # ranking : lower the better. ranking starting from 0
+
+    print("- Epoch starts from 0")
+    print("=======================================")
+    
+    list_level_1 = list()
+    for ep in range(len(list_validation_loss)):
+
+        local_model_ft = utils_sol.load_model_ft_with_epoch(
+            class_model_ft=class_model_ft, 
+            target_epoch=ep,
+            dir_model_ft=dir_model_ft,
+            name_model_ft=name_model_ft
+        )
+        
+        result = trainer.predict(local_model_ft, data_module)
+        result_pred = list()
+        result_label = list()
+        for bat in range(len(result)):
+            result_pred.append(result[bat][0].squeeze())
+            result_label.append(result[bat][1])
+
+        list_local_model_ft_result = utils_sol.model_evalulator_sol(
+            array_predictions=np.vstack(result_pred),
+            array_labels=np.vstack(result_label),
+            # dataset_dict=dataset_dict, 
+            solute_or_solvent=solute_or_solvent,
+            show_plot=False,
+            print_result=False,
+        )
+        # dataset_name, task, RMSE, MAE, p_value mantissam, p_value exponent
+        
+        # add epoch (starting from 0) to the right 
+        list_local_model_ft_result.append(ep)
+        # dataset_name, task, metric1 (RMSE or ROC-AUC), metric2 (MAE or None), p_value mantissam, p_value exponent, epoch
+        
+        list_level_1.append(list_local_model_ft_result)
+    print("=======================================")
+    print("=======================================")
+
+    # to get the metric_1 ranking
+    array_level_1 = np.array(list_level_1)
+    array_metric_1 = array_level_1[:, 2].astype('float32')
+    array_metric_1_ranking = utils_sol.rank_value_sol(list_value=array_metric_1,
+                                              # dataset_dict=dataset_dict,
+                                              is_loss=False)
+
+    # add loss, and ranking of the loss value to the right
+    # reg: lower the better, class: higher the better
+    array_level_1 = np.hstack((list_level_1, 
+                               np.expand_dims(list_validation_loss, axis=1), 
+                               np.expand_dims(array_loss_ranking, axis=1),
+                               np.expand_dims(array_metric_1_ranking, axis=1))) 
+    # solute_or_solvent, RMSE, MAE, p_value mantissam, p_value exponent, epoch, loss, loss_ranking, metric_1_ranking
+    
+    return array_level_1 
+    #################################### EX #########################################
+    # list_column_names = ['solute_or_solvent', 
+    #                      'metric_1', 
+    #                      'metric_2', 
+    #                      'p_value_mantissa', 
+    #                      'p_value_exponent', 
+    #                      'epoch', 
+    #                      'loss',
+    #                      'loss_ranking',
+    #                      'metric_1_ranking']
+    # df_evaluation_level_1 = pd.DataFrame(array_level_1, columns=list_column_names)
+    #################################################################################

chemllama_mtr.py

@@ -0,0 +1,212 @@
+import lightning as L
+import torch
+import torchmetrics
+
+from torch import nn
+from transformers import LlamaModel, LlamaConfig
+from pl_bolts.optimizers.lr_scheduler import LinearWarmupCosineAnnealingLR
+
+class ChemLlama(L.LightningModule):
+    def __init__(
+        self,
+        max_position_embeddings,
+        vocab_size,
+        pad_token_id,
+        bos_token_id,
+        eos_token_id,
+        steps_per_epoch=None, #
+        warmup_epochs=None, #
+        max_epochs=None, #
+        hidden_size=384,
+        intermediate_size=464,
+        num_labels=105,
+        attention_dropout=0.144,
+        num_hidden_layers=3,
+        num_attention_heads=12,
+        learning_rate=0.0001,
+    ):
+        super(ChemLlama, self).__init__()
+        self.save_hyperparameters()
+        
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_labels = num_labels
+        self.vocab_size = vocab_size
+        self.pad_token_id = pad_token_id
+        self.bos_token_id = bos_token_id
+        self.eos_token_id = eos_token_id
+        self.steps_per_epoch = steps_per_epoch #
+        self.warmup_epochs = warmup_epochs #
+        self.max_epochs = max_epochs #
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.attention_dropout = attention_dropout
+        self.max_position_embeddings = max_position_embeddings
+        self.learning_rate = learning_rate
+
+        self.mae = torchmetrics.MeanAbsoluteError()
+        self.mse = torchmetrics.MeanSquaredError()
+
+        self.config_llama = LlamaConfig(
+            max_position_embeddings=self.max_position_embeddings,
+            vocab_size=self.vocab_size,
+            hidden_size=self.hidden_size,
+            intermediate_size=self.intermediate_size,
+            num_hidden_layers=self.num_hidden_layers,
+            num_attention_heads=self.num_attention_heads,
+            attention_dropout=self.attention_dropout,
+            pad_token_id=self.pad_token_id,
+            bos_token_id=self.bos_token_id,
+            eos_token_id=self.eos_token_id,
+        )
+
+        self.loss_fn = nn.L1Loss()
+
+        self.llama = LlamaModel(self.config_llama)
+        self.gelu = nn.GELU()
+        self.score = nn.Linear(self.hidden_size, self.num_labels)
+
+    def forward(self, input_ids, attention_mask, labels=None):
+
+        transformer_outputs = self.llama(
+            input_ids=input_ids, attention_mask=attention_mask
+        )
+
+        hidden_states = transformer_outputs[0]
+        hidden_states = self.gelu(hidden_states)
+        logits = self.score(hidden_states)
+
+        if input_ids is not None:
+            batch_size = input_ids.shape[0]
+        else:
+            batch_size = inputs_embeds.shape[0]
+
+        if self.config_llama.pad_token_id is None and batch_size != 1:
+            raise ValueError(
+                "Cannot handle batch sizes > 1 if no padding token is defined."
+            )
+        if self.config_llama.pad_token_id is None:
+            sequence_lengths = -1
+        else:
+            if input_ids is not None:
+                # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility
+                sequence_lengths = (
+                    torch.eq(input_ids, self.config_llama.pad_token_id).int().argmax(-1)
+                    - 1
+                )
+                sequence_lengths = sequence_lengths % input_ids.shape[-1]
+                sequence_lengths = sequence_lengths.to(logits.device)
+            else:
+                sequence_lengths = -1
+        # raise ValueError(len(sequence_lengths), sequence_lengths)
+
+        pooled_logits = logits[
+            torch.arange(batch_size, device=logits.device), sequence_lengths
+        ]
+        return pooled_logits
+
+    def training_step(self, batch, batch_idx):
+
+        loss, logits, labels = self._common_step(batch=batch, batch_idx=batch_idx)
+
+        # mae = self.mae(logits, labels)
+        # mse = self.mse(logits, labels)
+        self.log_dict(
+            {
+                "train_loss": loss, 
+                # "train_mae": mae, 
+                # "train_mse": mse
+            },
+            on_step=True,
+            on_epoch=True,
+            prog_bar=True,
+            sync_dist=True,
+            # logger=True,
+        )
+        # on_stecp = True will use lots of computational resources
+
+        # return loss
+        return {"loss": loss, "logits": logits, "labels": labels}
+
+    def train_epoch_end(self, outputs):
+        # avg_loss = torch.stack([x["loss"] for x in outputs]).mean()
+        scores = torch.cat([x["logits"] for x in outputs])
+        labels = torch.cat([x["labels"] for x in outputs])
+        self.log_dict(
+            {
+                "train_mae": self.mae(scores, labels),
+                "train_mse": self.mse(scores, labels)
+            },
+            on_step=True,
+            on_epoch=True,
+            prog_bar=True,
+            sync_dist=True,
+        )
+
+    def validation_step(self, batch, batch_idx):
+
+        loss, logits, labels = self._common_step(batch=batch, batch_idx=batch_idx)
+        # self.log("val_loss", loss)
+        self.log("val_loss", loss, sync_dist=True)
+        return loss
+
+    def test_step(self, batch, batch_idx):
+
+        loss, logits, labels = self._common_step(batch=batch, batch_idx=batch_idx)
+        # self.log("val_loss", loss)
+        self.log("test_loss", loss, sync_dist=True,)
+        return loss
+
+    def _common_step(self, batch, batch_idx):
+
+        logits = self.forward(
+            input_ids=batch["input_ids"].squeeze(),
+            attention_mask=batch["attention_mask"].squeeze(),
+        )
+
+        labels = batch["labels"].squeeze()
+        loss = self.loss_fn(logits, labels)
+
+        # print(f"logits : {logits.shape}")
+        # print(f"labels : {labels.shape}")
+
+        return loss, logits, labels
+
+    # def configure_optimizers(self):  # Schedular here too!
+    #     # since confiture_optimizers and the model are included in the same class.. self.parameters()
+    #     return torch.optim.AdamW(
+    #         params=self.parameters(),
+    #         lr=self.learning_rate,
+    #         betas=(0.9, 0.999),
+    #         weight_decay=0.01,
+    #     )
+
+    # # The below is for warm-up scheduler
+    # https://lightning.ai/forums/t/how-to-use-warmup-lr-cosineannealinglr-in-lightning/1980
+    # https://github.com/Lightning-AI/pytorch-lightning/issues/328
+    def configure_optimizers(self):  # Schedular here too!
+        # since confiture_optimizers and the model are included in the same class.. self.parameters()
+        optimizer = torch.optim.AdamW(
+            params=self.parameters(),
+            lr=self.learning_rate,
+            betas=(0.9, 0.999),
+            weight_decay=0.01,
+        )
+
+        # "warmup_epochs //4 only not max_epochs" will work
+        scheduler = LinearWarmupCosineAnnealingLR(
+            optimizer, 
+            warmup_epochs=self.warmup_epochs*self.steps_per_epoch // torch.cuda.device_count(), # // num_device in this case
+            max_epochs=self.max_epochs*self.steps_per_epoch // torch.cuda.device_count(),
+        )
+
+        return {
+            "optimizer": optimizer,
+            "lr_scheduler": {
+                "scheduler": scheduler,
+                "interval": "step",
+                "frequency": 1,
+                "reduce_on_plateau": False,
+                "monitor": "val_loss",
+            }
+        }

datamodule_finetune_sl.py

@@ -0,0 +1,120 @@
+import lightning as L
+import torch
+import numpy as np
+import pandas as pd
+from sklearn.model_selection import train_test_split
+
+from torch.utils.data import Dataset, DataLoader
+from transformers import DataCollatorWithPadding
+
+
+class CustomLlamaDatasetAbraham(Dataset):
+    def __init__(self, df, tokenizer, max_seq_length):
+        self.keys = df.iloc[:, 0] # 1D array
+        self.labels = df.iloc[:, 1:] # 2D array
+        self.tokenizer = tokenizer
+        self.max_seq_length = max_seq_length
+
+    def __len__(self):
+        return self.keys.shape[0]
+
+    def fn_token_encode(self, smiles):
+        return self.tokenizer(
+            smiles,
+            truncation=True,
+            padding="max_length",
+            max_length=self.max_seq_length,
+        )
+
+    def __getitem__(self, idx):
+        local_encoded = self.fn_token_encode(self.keys.iloc[idx])
+        
+        return {
+            "input_ids": torch.tensor(local_encoded["input_ids"]),
+            "attention_mask": torch.tensor(local_encoded["attention_mask"]),
+            "labels": torch.tensor(self.labels.iloc[idx]),
+        }
+
+
+class CustomFinetuneDataModule(L.LightningDataModule):
+    def __init__(
+        self,
+        solute_or_solvent,
+        tokenizer,
+        max_seq_length,
+        batch_size_train,
+        batch_size_valid,
+        num_device,
+    ):
+        super().__init__()
+
+        self.solute_or_solvent = solute_or_solvent
+        self.tokenizer = tokenizer
+        self.max_seq_length = max_seq_length
+        self.batch_size_train = batch_size_train
+        self.batch_size_valid = batch_size_valid
+        self.data_collator = DataCollatorWithPadding(self.tokenizer)
+        self.num_device = num_device
+    
+
+    def prepare_data(self, smiles_str:str):
+        # self.list_df = load_abraham(self.solute_or_solvent)
+        self.smiles_str = smiles_str
+
+    def setup(self, stage=None):
+        # self.train_df, self.valid_df, self.test_df = self.list_df
+        self.train_df = None
+        self.valid_df = None
+        self.test_df = self.smiles_str
+
+    def train_dataloader(self):
+        return DataLoader(
+            dataset=CustomLlamaDatasetAbraham(
+                self.train_df, self.tokenizer, self.max_seq_length,
+            ),
+            batch_size=self.batch_size_train,
+            num_workers=self.num_device,
+            collate_fn=self.data_collator,
+            shuffle=True,
+        )
+
+    def val_dataloader(self):
+        return DataLoader(
+            dataset=CustomLlamaDatasetAbraham(
+                self.valid_df, self.tokenizer, self.max_seq_length,
+            ),
+            batch_size=self.batch_size_valid,
+            num_workers=self.num_device,
+            collate_fn=self.data_collator,
+            shuffle=False,
+        )
+
+    def test_dataloader(self):
+        return DataLoader(
+            dataset=CustomLlamaDatasetAbraham(
+                self.test_df, self.tokenizer, self.max_seq_length,
+            ),
+            batch_size=self.batch_size_valid,
+            num_workers=self.num_device,
+            collate_fn=self.data_collator,
+            shuffle=False,
+        )
+        
+    # It uses test_df
+    def predict_dataloader(self): 
+        return DataLoader(
+            dataset=CustomLlamaDatasetAbraham(
+                self.test_df, self.tokenizer, self.max_seq_length,
+            ),
+            batch_size=self.batch_size_valid,
+            num_workers=self.num_device,
+            collate_fn=self.data_collator,
+            shuffle=False,
+        )
+
+
+
+
+
+
+        

dict_dtype_slpy

@@ -0,0 +1,9 @@
+dict_dtype = {
+    "solute_or_solvent": "str",
+    "metric_1": "float32",
+    "metric_2": "float32",
+    "epoch": "int32",
+    "loss": "float32",
+    "loss_ranking": "int32",
+    "metric_1_ranking": "int32",
+}

model_finetune_sl.py

@@ -0,0 +1,152 @@
+import torch
+from torch import nn
+import lightning as L
+from pl_bolts.optimizers.lr_scheduler import LinearWarmupCosineAnnealingLR
+import numpy as np
+
+class CustomFinetuneModel(L.LightningModule):
+    def __init__(
+        self,
+        model_mtr,
+        steps_per_epoch, #
+        warmup_epochs, #
+        max_epochs, #
+        learning_rate,
+        linear_param:int=64,
+        use_freeze:bool=True,
+        *args, **kwargs
+    ):
+        super(CustomFinetuneModel, self).__init__()
+        # self.save_hyperparameters()
+
+        self.model_mtr = model_mtr
+        if use_freeze:
+            self.model_mtr.freeze()
+            # for name, param in model_mtr.named_parameters():
+            #     param.requires_grad = False
+            #     print(name, param.requires_grad)
+
+        self.steps_per_epoch = steps_per_epoch
+        self.warmup_epochs = warmup_epochs
+        self.max_epochs = max_epochs
+        self.learning_rate = learning_rate
+
+        self.list_val_loss = list()
+        
+        self.gelu = nn.GELU()
+        self.linear1 = nn.Linear(self.model_mtr.num_labels, linear_param)
+        self.linear2 = nn.Linear(linear_param, linear_param)
+        self.regression = nn.Linear(linear_param, 5)
+
+        self.loss_fn = nn.L1Loss()
+
+    def forward(self, input_ids, attention_mask, labels=None):
+        x = self.model_mtr(input_ids=input_ids, attention_mask=attention_mask)
+        x = self.gelu(x)
+        x = self.linear1(x)
+        x = self.gelu(x)
+        x = self.linear2(x)
+        x = self.gelu(x)
+        x = self.regression(x)
+        
+        return x
+
+    def training_step(self, batch, batch_idx):
+
+        loss, logits, labels = self._common_step(batch=batch, batch_idx=batch_idx)
+
+        self.log_dict(
+            {
+                "train_loss": loss, 
+            },
+            on_step=True,
+            on_epoch=True,
+            prog_bar=True,
+            # sync_dist=True,
+        )
+
+        return {"loss": loss, "logits": logits, "labels": labels}
+
+    def validation_step(self, batch, batch_idx):
+
+        loss, logits, labels = self._common_step(batch=batch, batch_idx=batch_idx)
+        # self.log("val_loss", loss)
+        self.log("val_loss", loss, sync_dist=True)
+        
+        return loss
+
+    def valid_epoch_end(self, outputs):
+        # avg_loss = torch.stack([x["loss"] for x in outputs]).mean()
+        scores = torch.cat([x["logits"] for x in outputs])
+        labels = torch.cat([x["labels"] for x in outputs])
+        self.list_val_loss.append(self.loss_fn(scores, labels))
+        self.log_dict(
+            {
+                "list_val_loss": self.list_val_loss,
+            },
+            on_step=False,
+            on_epoch=True,
+            prog_bar=True,
+            # sync_dist=True,
+        )
+    
+    # def get_val_loss_history(self):
+    #     return np.array(self.list_val_loss).squeeze()
+        
+    def test_step(self, batch, batch_idx):
+
+        loss, logits, labels = self._common_step(batch=batch, batch_idx=batch_idx)
+        # self.log("val_loss", loss)
+        self.log("test_loss", loss, sync_dist=True,)
+        
+        return loss
+
+    def _common_step(self, batch, batch_idx):
+
+        logits = self.forward(
+            input_ids=batch["input_ids"].squeeze(),
+            attention_mask=batch["attention_mask"].squeeze(),
+        ).squeeze()
+
+        labels = batch["labels"]
+        loss = self.loss_fn(logits, labels)
+
+        return loss, logits, labels
+
+    def predict_step(self, batch, batch_idx):
+        loss, logits, labels = self._common_step(batch=batch, batch_idx=batch_idx)
+        
+        return logits, labels
+
+    def configure_optimizers(self):  # Schedular here too!
+        # since confiture_optimizers and the model are included in the same class.. self.parameters()
+        optimizer = torch.optim.AdamW(
+            params=self.parameters(),
+            lr=self.learning_rate,
+            betas=(0.9, 0.999),
+            weight_decay=0.01,
+        )
+
+        # "warmup_epochs //4 only not max_epochs" will work
+        scheduler = LinearWarmupCosineAnnealingLR(
+            optimizer, 
+            # warmup_epochs=self.warmup_epochs*self.steps_per_epoch // 4, # // num_device in this case
+            # max_epochs=self.max_epochs*self.steps_per_epoch // 4,
+            # Better not to use Multiple GPUs due to the smaller dataset size.
+            warmup_epochs=self.warmup_epochs*self.steps_per_epoch, # // num_device in this case
+            max_epochs=self.max_epochs*self.steps_per_epoch,
+        )
+
+        return {
+            "optimizer": optimizer,
+            "lr_scheduler": {
+                "scheduler": scheduler,
+                "interval": "step",
+                "frequency": 1,
+                "reduce_on_plateau": False,
+                "monitor": "val_loss",
+            }
+        }
+
+    
+        

requirements.txt

@@ -0,0 +1,12 @@
+torch
+transformers
+lightning
+lightning-bolts
+numpy
+pytorch-ignite
+pytorch-lightning
+pytorch-lightning-bolts
+pytorch-warmup
+scikit-learn
+scipy
+seaborn

run_auto_llama_cuda0.py

@@ -0,0 +1,122 @@
+import sys
+import os
+
+# This means you will use the first GPU among the four GPUs in our case.
+# "0", "1", "2", "3". Since FT dataset is small, using one GPU should be proper.
+os.environ["CUDA_VISIBLE_DEVICES"]= "0"
+
+
+import torch
+import numpy as np
+import pandas as pd
+import warnings
+import lightning as L
+torch.set_float32_matmul_precision('high')
+
+# Filter out FutureWarning and UnderReviewWarning messages from pl_bolts
+warnings.filterwarnings("ignore", module="pl_bolts")
+
+# Add the parent directory to sys.path
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+import tokenizer_sl
+import auto_evaluator_sl
+
+print(os.path.dirname(__file__))
+
+
+torch.manual_seed(1004)
+np.random.seed(1004)
+
+print(os.getcwd())
+
+"""
+Note 1 to Dr. Lang
+
+I have checked that when we not freeze the MTR model, then the test loss values are keep decreasing when I set the epochs as 7.
+(At least for solute.) So We you may try to run more epochs if you want. But Solvent may be already overfitted or will get soon since it has only few data.
+
+Using learning rate bigger than the default setting is not that recommanded since we don't freeze the MTR model.
+But lower lr could work.
+
+Be aware of doing version control (ver_ft). Make sure you keep the same version for both 'solute' and 'solvent' otherwise, you will get confused.
+
+The variable "dir_model_ft_to_save" is where the FT model get saved.
+The result csv files will be located at 'evaluations/corresponding version/solute and (or) solvent.csv'
+
+You can run this code by
+cd ~/SolLlama
+python run_auto_llama_cuda0.py
+
+But makes sure you are running this in your virtual environment that all requirements_cuda118.txt installed
+"""
+
+
+"""
+# You can run both 'solute' and 'solvent' at one run by doing the below
+for solute_or_solvent in ['solute' ,'solvent']:
+    The REST of the codes except the variant solute_or_solvent right below with this (SAME) indentation levels
+"""
+#### Hyper Parameters ##### <- You can control these parameters as you want
+# solute_or_solvent = 'solvent'
+solute_or_solvent = 'solute'
+ver_ft = 0 # version control for FT model & evaluation data # Or it will overwrite the models and results
+batch_size_pair = [64, 64] if solute_or_solvent == 'solute' else [10, 10] # [train, valid(test)] 
+# since 'solute' has very small dataset. So I thinl 10 for train and 10 for valid(test) should be the maximum values.
+lr = 0.0001 
+epochs = 7
+use_freeze = False  # Freeze the model or not # False measn not freezing
+overwrite_level_2 = True # If you don't want to overwrite the models and csv files, then change this to False
+###########################
+
+
+# I just reused our previous research code with some modifications.
+dir_main = "/home/ylee/SolLlama"
+name_model_mtr = "ChemLlama_Medium_30m_vloss_val_loss=0.029_ep_epoch=04.ckpt" 
+
+dir_model_mtr = f"{dir_main}/model_mtr/{name_model_mtr}"
+
+max_seq_length = 512
+
+tokenizer = tokenizer_sol.fn_load_tokenizer_llama(
+    max_seq_length=max_seq_length,
+)
+max_length = max_seq_length
+num_workers = 2
+
+dir_model_ft_to_save = f"{dir_main}/save_models_ft/ft_version_{ver_ft}"
+
+array_level_2 = auto_evaluator_sol.auto_evaluator_level_2_sol(
+    dir_model_mtr=dir_model_mtr,
+    dir_model_ft_to_save=dir_model_ft_to_save,
+    tokenizer=tokenizer,
+    max_length=max_seq_length,
+    solute_or_solvent=solute_or_solvent,
+    num_workers=num_workers,
+    batch_size_pair=batch_size_pair,
+    lr=lr,
+    overwrite_level_2=overwrite_level_2,
+    epochs=epochs,
+    use_freeze=use_freeze,
+)
+
+print(array_level_2.shape)
+print(array_level_2)
+
+list_column_names_level_2 = [
+    'solute_or_solvent', 
+    'metric_1', 
+    'metric_2', 
+    'epoch', 
+    'loss',
+    'loss_ranking',
+    'metric_1_ranking'
+]
+
+df_evaluation_level_2 = pd.DataFrame(array_level_2, columns=list_column_names_level_2)
+
+os.makedirs(f'{os.path.dirname(__file__)}/evaluations/ft_version_{ver_ft}', exist_ok=True)
+df_evaluation_level_2.to_csv(f'{os.path.dirname(__file__)}/evaluations/ft_version_{ver_ft}/{solute_or_solvent}.csv', index=False)
+
+
+

tokenizer.json

@@ -0,0 +1 @@
+{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":0,"special":true,"content":"<pad>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":11,"special":true,"content":"<unk>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":12,"special":true,"content":"<s>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":13,"special":true,"content":"</s>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],"normalizer":null,"pre_tokenizer":{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":true},"post_processor":{"type":"RobertaProcessing","sep":["[SEP]",13],"cls":["[CLS]",12],"trim_offsets":true,"add_prefix_space":false},"decoder":{"type":"ByteLevel","add_prefix_space":true,"trim_offsets":true},"model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"vocab":{"<pad>":0,"[unused1]":1,"[unused2]":2,"[unused3]":3,"[unused4]":4,"[unused5]":5,"[unused6]":6,"[unused7]":7,"[unused8]":8,"[unused9]":9,"[unused10]":10,"<unk>":11,"<s>":12,"</s>":13,"<mask>":14,"c":15,"C":16,"(":17,")":18,"O":19,"1":20,"2":21,"=":22,"N":23,".":24,"n":25,"3":26,"F":27,"Cl":28,">>":29,"~":30,"-":31,"4":32,"[C@H]":33,"S":34,"[C@@H]":35,"[O-]":36,"Br":37,"#":38,"/":39,"[nH]":40,"[N+]":41,"s":42,"5":43,"o":44,"P":45,"[Na+]":46,"[Si]":47,"I":48,"[Na]":49,"[Pd]":50,"[K+]":51,"[K]":52,"[P]":53,"B":54,"[C@]":55,"[C@@]":56,"[Cl-]":57,"6":58,"[OH-]":59,"\\":60,"[N-]":61,"[Li]":62,"[H]":63,"[2H]":64,"[NH4+]":65,"[c-]":66,"[P-]":67,"[Cs+]":68,"[Li+]":69,"[Cs]":70,"[NaH]":71,"[H-]":72,"[O+]":73,"[BH4-]":74,"[Cu]":75,"7":76,"[Mg]":77,"[Fe+2]":78,"[n+]":79,"[Sn]":80,"[BH-]":81,"[Pd+2]":82,"[CH]":83,"[I-]":84,"[Br-]":85,"[C-]":86,"[Zn]":87,"[B-]":88,"[F-]":89,"[Al]":90,"[P+]":91,"[BH3-]":92,"[Fe]":93,"[C]":94,"[AlH4]":95,"[Ni]":96,"[SiH]":97,"8":98,"[Cu+2]":99,"[Mn]":100,"[AlH]":101,"[nH+]":102,"[AlH4-]":103,"[O-2]":104,"[Cr]":105,"[Mg+2]":106,"[NH3+]":107,"[S@]":108,"[Pt]":109,"[Al+3]":110,"[S@@]":111,"[S-]":112,"[Ti]":113,"[Zn+2]":114,"[PH]":115,"[NH2+]":116,"[Ru]":117,"[Ag+]":118,"[S+]":119,"[I+3]":120,"[NH+]":121,"[Ca+2]":122,"[Ag]":123,"9":124,"[Os]":125,"[Se]":126,"[SiH2]":127,"[Ca]":128,"[Ti+4]":129,"[Ac]":130,"[Cu+]":131,"[S]":132,"[Rh]":133,"[Cl+3]":134,"[cH-]":135,"[Zn+]":136,"[O]":137,"[Cl+]":138,"[SH]":139,"[H+]":140,"[Pd+]":141,"[se]":142,"[PH+]":143,"[I]":144,"[Pt+2]":145,"[C+]":146,"[Mg+]":147,"[Hg]":148,"[W]":149,"[SnH]":150,"[SiH3]":151,"[Fe+3]":152,"[NH]":153,"[Mo]":154,"[CH2+]":155,"%10":156,"[CH2-]":157,"[CH2]":158,"[n-]":159,"[Ce+4]":160,"[NH-]":161,"[Co]":162,"[I+]":163,"[PH2]":164,"[Pt+4]":165,"[Ce]":166,"[B]":167,"[Sn+2]":168,"[Ba+2]":169,"%11":170,"[Fe-3]":171,"[18F]":172,"[SH-]":173,"[Pb+2]":174,"[Os-2]":175,"[Zr+4]":176,"[N]":177,"[Ir]":178,"[Bi]":179,"[Ni+2]":180,"[P@]":181,"[Co+2]":182,"[s+]":183,"[As]":184,"[P+3]":185,"[Hg+2]":186,"[Yb+3]":187,"[CH-]":188,"[Zr+2]":189,"[Mn+2]":190,"[CH+]":191,"[In]":192,"[KH]":193,"[Ce+3]":194,"[Zr]":195,"[AlH2-]":196,"[OH2+]":197,"[Ti+3]":198,"[Rh+2]":199,"[Sb]":200,"[S-2]":201,"%12":202,"[P@@]":203,"[Si@H]":204,"[Mn+4]":205,"p":206,"[Ba]":207,"[NH2-]":208,"[Ge]":209,"[Pb+4]":210,"[Cr+3]":211,"[Au]":212,"[LiH]":213,"[Sc+3]":214,"[o+]":215,"[Rh-3]":216,"%13":217,"[Br]":218,"[Sb-]":219,"[S@+]":220,"[I+2]":221,"[Ar]":222,"[V]":223,"[Cu-]":224,"[Al-]":225,"[Te]":226,"[13c]":227,"[13C]":228,"[Cl]":229,"[PH4+]":230,"[SiH4]":231,"[te]":232,"[CH3-]":233,"[S@@+]":234,"[Rh+3]":235,"[SH+]":236,"[Bi+3]":237,"[Br+2]":238,"[La]":239,"[La+3]":240,"[Pt-2]":241,"[N@@]":242,"[PH3+]":243,"[N@]":244,"[Si+4]":245,"[Sr+2]":246,"[Al+]":247,"[Pb]":248,"[SeH]":249,"[Si-]":250,"[V+5]":251,"[Y+3]":252,"[Re]":253,"[Ru+]":254,"[Sm]":255,"*":256,"[3H]":257,"[NH2]":258,"[Ag-]":259,"[13CH3]":260,"[OH+]":261,"[Ru+3]":262,"[OH]":263,"[Gd+3]":264,"[13CH2]":265,"[In+3]":266,"[Si@@]":267,"[Si@]":268,"[Ti+2]":269,"[Sn+]":270,"[Cl+2]":271,"[AlH-]":272,"[Pd-2]":273,"[SnH3]":274,"[B+3]":275,"[Cu-2]":276,"[Nd+3]":277,"[Pb+3]":278,"[13cH]":279,"[Fe-4]":280,"[Ga]":281,"[Sn+4]":282,"[Hg+]":283,"[11CH3]":284,"[Hf]":285,"[Pr]":286,"[Y]":287,"[S+2]":288,"[Cd]":289,"[Cr+6]":290,"[Zr+3]":291,"[Rh+]":292,"[CH3]":293,"[N-3]":294,"[Hf+2]":295,"[Th]":296,"[Sb+3]":297,"%14":298,"[Cr+2]":299,"[Ru+2]":300,"[Hf+4]":301,"[14C]":302,"[Ta]":303,"[Tl+]":304,"[B+]":305,"[Os+4]":306,"[PdH2]":307,"[Pd-]":308,"[Cd+2]":309,"[Co+3]":310,"[S+4]":311,"[Nb+5]":312,"[123I]":313,"[c+]":314,"[Rb+]":315,"[V+2]":316,"[CH3+]":317,"[Ag+2]":318,"[cH+]":319,"[Mn+3]":320,"[Se-]":321,"[As-]":322,"[Eu+3]":323,"[SH2]":324,"[Sm+3]":325,"[IH+]":326,"%15":327,"[OH3+]":328,"[PH3]":329,"[IH2+]":330,"[SH2+]":331,"[Ir+3]":332,"[AlH3]":333,"[Sc]":334,"[Yb]":335,"[15NH2]":336,"[Lu]":337,"[sH+]":338,"[Gd]":339,"[18F-]":340,"[SH3+]":341,"[SnH4]":342,"[TeH]":343,"[Si@@H]":344,"[Ga+3]":345,"[CaH2]":346,"[Tl]":347,"[Ta+5]":348,"[GeH]":349,"[Br+]":350,"[Sr]":351,"[Tl+3]":352,"[Sm+2]":353,"[PH5]":354,"%16":355,"[N@@+]":356,"[Au+3]":357,"[C-4]":358,"[Nd]":359,"[Ti+]":360,"[IH]":361,"[N@+]":362,"[125I]":363,"[Eu]":364,"[Sn+3]":365,"[Nb]":366,"[Er+3]":367,"[123I-]":368,"[14c]":369,"%17":370,"[SnH2]":371,"[YH]":372,"[Sb+5]":373,"[Pr+3]":374,"[Ir+]":375,"[N+3]":376,"[AlH2]":377,"[19F]":378,"%18":379,"[Tb]":380,"[14CH]":381,"[Mo+4]":382,"[Si+]":383,"[BH]":384,"[Be]":385,"[Rb]":386,"[pH]":387,"%19":388,"%20":389,"[Xe]":390,"[Ir-]":391,"[Be+2]":392,"[C+4]":393,"[RuH2]":394,"[15NH]":395,"[U+2]":396,"[Au-]":397,"%21":398,"%22":399,"[Au+]":400,"[15n]":401,"[Al+2]":402,"[Tb+3]":403,"[15N]":404,"[V+3]":405,"[W+6]":406,"[14CH3]":407,"[Cr+4]":408,"[ClH+]":409,"b":410,"[Ti+6]":411,"[Nd+]":412,"[Zr+]":413,"[PH2+]":414,"[Fm]":415,"[N@H+]":416,"[RuH]":417,"[Dy+3]":418,"%23":419,"[Hf+3]":420,"[W+4]":421,"[11C]":422,"[13CH]":423,"[Er]":424,"[124I]":425,"[LaH]":426,"[F]":427,"[siH]":428,"[Ga+]":429,"[Cm]":430,"[GeH3]":431,"[IH-]":432,"[U+6]":433,"[SeH+]":434,"[32P]":435,"[SeH-]":436,"[Pt-]":437,"[Ir+2]":438,"[se+]":439,"[U]":440,"[F+]":441,"[BH2]":442,"[As+]":443,"[Cf]":444,"[ClH2+]":445,"[Ni+]":446,"[TeH3]":447,"[SbH2]":448,"[Ag+3]":449,"%24":450,"[18O]":451,"[PH4]":452,"[Os+2]":453,"[Na-]":454,"[Sb+2]":455,"[V+4]":456,"[Ho+3]":457,"[68Ga]":458,"[PH-]":459,"[Bi+2]":460,"[Ce+2]":461,"[Pd+3]":462,"[99Tc]":463,"[13C@@H]":464,"[Fe+6]":465,"[c]":466,"[GeH2]":467,"[10B]":468,"[Cu+3]":469,"[Mo+2]":470,"[Cr+]":471,"[Pd+4]":472,"[Dy]":473,"[AsH]":474,"[Ba+]":475,"[SeH2]":476,"[In+]":477,"[TeH2]":478,"[BrH+]":479,"[14cH]":480,"[W+]":481,"[13C@H]":482,"[AsH2]":483,"[In+2]":484,"[N+2]":485,"[N@@H+]":486,"[SbH]":487,"[60Co]":488,"[AsH4+]":489,"[AsH3]":490,"[18OH]":491,"[Ru-2]":492,"[Na-2]":493,"[CuH2]":494,"[31P]":495,"[Ti+5]":496,"[35S]":497,"[P@@H]":498,"[ArH]":499,"[Co+]":500,"[Zr-2]":501,"[BH2-]":502,"[131I]":503,"[SH5]":504,"[VH]":505,"[B+2]":506,"[Yb+2]":507,"[14C@H]":508,"[211At]":509,"[NH3+2]":510,"[IrH]":511,"[IrH2]":512,"[Rh-]":513,"[Cr-]":514,"[Sb+]":515,"[Ni+3]":516,"[TaH3]":517,"[Tl+2]":518,"[64Cu]":519,"[Tc]":520,"[Cd+]":521,"[1H]":522,"[15nH]":523,"[AlH2+]":524,"[FH+2]":525,"[BiH3]":526,"[Ru-]":527,"[Mo+6]":528,"[AsH+]":529,"[BaH2]":530,"[BaH]":531,"[Fe+4]":532,"[229Th]":533,"[Th+4]":534,"[As+3]":535,"[NH+3]":536,"[P@H]":537,"[Li-]":538,"[7NaH]":539,"[Bi+]":540,"[PtH+2]":541,"[p-]":542,"[Re+5]":543,"[NiH]":544,"[Ni-]":545,"[Xe+]":546,"[Ca+]":547,"[11c]":548,"[Rh+4]":549,"[AcH]":550,"[HeH]":551,"[Sc+2]":552,"[Mn+]":553,"[UH]":554,"[14CH2]":555,"[SiH4+]":556,"[18OH2]":557,"[Ac-]":558,"[Re+4]":559,"[118Sn]":560,"[153Sm]":561,"[P+2]":562,"[9CH]":563,"[9CH3]":564,"[Y-]":565,"[NiH2]":566,"[Si+2]":567,"[Mn+6]":568,"[ZrH2]":569,"[C-2]":570,"[Bi+5]":571,"[24NaH]":572,"[Fr]":573,"[15CH]":574,"[Se+]":575,"[At]":576,"[P-3]":577,"[124I-]":578,"[CuH2-]":579,"[Nb+4]":580,"[Nb+3]":581,"[MgH]":582,"[Ir+4]":583,"[67Ga+3]":584,"[67Ga]":585,"[13N]":586,"[15OH2]":587,"[2NH]":588,"[Ho]":589,"[Cn]":590},"merges":[]}}

tokenizer_sl.py

@@ -0,0 +1,35 @@
+from transformers import LlamaTokenizerFast
+import json
+import os
+
+
+def fn_load_tokenizer_llama(
+    max_seq_length,
+    dir_tokenizer: str = "./tokenizer.json",
+    # dir_tokenizer:str = os.path.abspath(os.path.join(os.getcwd(), '..', "models_mtr/tokenizer.json")), # for JUP
+    add_eos_token:bool = True,
+):
+
+    tokenizer = LlamaTokenizerFast(
+        tokenizer_file=dir_tokenizer,
+        model_max_length=max_seq_length,
+        padding_side="right",
+        bos_token="<s>",
+        eos_token="</s>",
+        unk_token="<unk>",
+        add_eos_token=add_eos_token,
+    )
+    tokenizer.add_special_tokens({"pad_token": "<pad>", "sep_token": "</s>", "cls_token": "<s>", "mask_token":"<mask>"})
+    # tokenizer.add_special_tokens({"pad_token": "<pad>"})
+
+    return tokenizer
+
+def fn_load_descriptor_list(
+    key_descriptor_list,
+    dir_descriptor_list,
+):
+
+    with open(dir_descriptor_list, "r") as js:
+        list_descriptor = json.load(js)[key_descriptor_list]
+
+    return list_descriptor

utils_sl.py

@@ -0,0 +1,114 @@
+from sklearn.metrics import roc_curve, auc, roc_auc_score
+from sklearn.metrics import mean_squared_error
+from sklearn.metrics import r2_score
+from sklearn.metrics import mean_absolute_error
+from scipy.stats import spearmanr
+import matplotlib.pyplot as plt
+
+import numpy as np
+import os
+
+def model_evalulator_sol(
+    array_predictions, 
+    array_labels, 
+    # dataset_dict:dict, 
+    solute_or_solvent:str,
+    show_plot:bool=True, 
+    print_result:bool=True,
+):
+    
+    if print_result:        
+        print(f"Dataset : {solute_or_solvent}")
+        print("N:", array_labels.shape[0])
+    
+    fig, ax = plt.subplots()
+
+    metric = mean_squared_error(array_labels, array_predictions, squared=False) #RMSE
+    r2 = r2_score(array_labels, array_predictions)
+    metric2 = mean_absolute_error(array_labels, array_predictions) # MAE
+    ax.scatter(array_labels, array_predictions)
+    ax.set_title("Scatter Plot of Labels vs Predictions")
+    ax.set_xlabel("Labels")
+    ax.set_ylabel("Predictions")
+
+    if print_result:
+        print("R2:", r2)
+        print("Root Mean Square Error:", metric)
+        print("Mean Absolute Error:", metric2)
+
+    # correlation, p_value = spearmanr(array_labels, array_predictions)
+
+    # if print_result:
+    #     print("Spearman correlation:", correlation)
+    #     print("p-value:", p_value)
+    #     print("=======================================")
+    
+    xmin, xmax = ax.get_xlim()
+    ax.set_ylim(xmin, xmax)
+    
+    if not show_plot:
+        plt.ioff()
+        plt.clf()
+        plt.close()
+    else :
+        plt.show()
+    
+    # metrict 1 - ROC score (classification) | RMSE (regression)
+    # metric 2 - None (classification) | MAE ( regression)
+    round_decimal = 6
+    if metric2 != None:
+        metric2 = round(metric2, round_decimal)
+
+    # list_p_value = str(p_value).split('e')
+    # p_value_mantissa = round(float(list_p_value[0]), round_decimal)
+    # if len(list_p_value) == 2:
+    #     p_value_exponent = int(list_p_value[1])
+    # else:
+    #     p_value_exponent = None
+
+    return [solute_or_solvent,
+            round(metric, round_decimal), 
+            metric2]
+    # return [solute_or_solvent,
+    #         round(metric, round_decimal), 
+    #         metric2,
+    #         p_value_mantissa,
+    #         p_value_exponent]
+
+# from .model_finetune import CustomFinetuneModel
+# import model_finetune_sol
+import torch
+def load_model_ft_with_epoch(class_model_ft,
+                             target_epoch:int,
+                             dir_model_ft:str,
+                             name_model_ft:str):
+    # dir_model_ft level 1
+    # ex /main/model_mtr/model_mtr_ep/dataset
+    
+    dir_all_model_ft = f"{dir_model_ft}/{name_model_ft}/version_0/checkpoints/"
+    list_files_in_dir_model_ft = os.listdir(dir_all_model_ft)
+    # extension = '.ckpt'
+    extension = '.pt'
+    list_model_ft_in_the_dir = sorted(list_files_in_dir_model_ft, key=lambda x: float(x.split('=')[-1].split('.')[0]))
+    
+    print(f"Loaded model with epoch {target_epoch}")
+    dir_target_model_ft = f"{dir_all_model_ft}/{list_model_ft_in_the_dir[target_epoch]}"
+    
+    # class_model_ft.load_from_checkpoint(dir_target_model_ft)
+    
+    loaded_state_dict = torch.load(dir_target_model_ft)
+    class_model_ft.load_state_dict(loaded_state_dict['state_dict'])
+    
+    return class_model_ft # now is model_ft
+
+from scipy.stats import rankdata
+# rankdata does not consider decimal places!
+def rank_value_sol(
+    list_value, 
+    # dataset_dict:dict, 
+    is_loss:bool=True,
+):
+    
+    list_value = np.array(list_value)
+
+    return np.array(rankdata(list_value * 100000, method='min')) - 1