initial upload

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tf_model.ckpt filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,168 @@
+# To run streamlit, go to terminal and type: 'streamlit run app.py'
+# Core Packages ###########################
+import os
+import math
+
+import numpy as np
+import streamlit as st
+import pandas as pd
+
+from model import BertLightningModel
+
+#######################################################################################################################
+current_path = os.path.abspath(os.path.dirname(__file__))
+
+project_title = "Automatic Essay Scoring"
+project_desc = "The Automatic Essay Scoring app is a tool that uses natural language processing and machine learning algorithms to automatically grade essays. " \
+               "This is an application of Microsoft's Deberta v3-large model."
+project_icon = "46_Knowledge-white4.png"
+
+project_link = "https://huggingface.co/microsoft/deberta-v3-large \n https://www.kaggle.com/code/yasufuminakama/fb3-deberta-v3-base-baseline-train/notebook"
+st.set_page_config(page_title=project_title, initial_sidebar_state='collapsed',page_icon=project_icon)
+
+# additional info from the readme
+add_info_md = """
+
+"""
+#######################################################################################################################
+def load_model():
+    CONFIG = dict(
+        model_name="microsoft/deberta-v3-large",
+        num_classes=6,
+        lr=2e-5,
+
+        batch_size=8,
+        num_workers=8,
+        max_length=512,
+        weight_decay=0.01,
+
+        accelerator='gpu',
+        max_epochs=5,
+        accumulate_grad_batches=4,
+        precision=16,
+        gradient_clip_val=1000,
+        train_size=0.8,
+        num_cross_val_splits=5,
+        num_frozen_layers=20,  # out of 24 in deberta
+    )
+    model = BertLightningModel.load_from_checkpoint(os.path.join(current_path,'tf_model.ckpt'),config=CONFIG, map_location='cpu')
+
+    return model
+
+def predict(_input, _model):
+    tokens = _model.tokenizer([_input], return_tensors='pt')
+    outputs = _model(tokens)[0].tolist()
+
+    df = pd.DataFrame({
+        'Criterion': ['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions'],
+        'Grade': outputs
+    })
+
+    return df
+
+def convert_ave_to_score_range(score, max, min):
+    fg = (score-1) * ((max-min)/3) + min
+    return fg
+
+def run_model(answer, min_score, max_score):
+    evaluation = 0
+    if len(answer) > 20:
+        model = load_model()
+        evaluation = predict(answer,model)
+    else:
+        st.error("Your answer is too short.")
+
+    # get the average of the score evaluations
+    ave = evaluation['Grade'].mean()
+
+    grade = convert_ave_to_score_range(ave,max_score,min_score)
+    grade = round(grade)
+    final_grade = max_score if max_score < grade else grade
+    return evaluation, final_grade
+
+def main():
+    head_col = st.columns([1,8])
+    with head_col[0]:
+        st.image(project_icon)
+    with head_col[1]:
+        st.title(project_title)
+    st.write(project_desc)
+    st.write(f"Source Project: {project_link}")
+    # expander = st.expander("Additional Information")
+    # expander.markdown(add_info_md)
+    st.markdown("***")
+    st.subheader("")
+#########################################
+    # instructions
+    st.subheader("How to use: \n"
+                 "1. Input your essay in the text box.\n"
+                 "2. Click on \'Grade Essay\' button to run the model.")
+
+#########################################
+
+    # text area input for the prompt and response, button to run the model, other widgets
+    response_ta = st.text_area("Essay:",placeholder="Input the answer / essay here.",height=500)
+    col1,col2,col3 = st.columns(3)
+    min_score = col1.number_input('Minimum Score',0,100,0)
+    max_score = col2.number_input('Maximum Score',0,100,10)
+    run_button = st.button("Grade Essay")
+
+    # button is clicked
+    if run_button:
+        if not response_ta: # if any text area is empty:
+            st.error("Please input the essay in their corresponding text area.")
+        if min_score >= max_score:
+            st.error("Minimum score must be less than maximum score.")
+        else: # run model
+            eval_df, score = run_model(answer=response_ta, min_score=min_score, max_score=max_score)
+            # output message template
+            msg = f"Your essay score is: {score} (Minimum Possible Score: {min_score} | Maximum Possible Score: {max_score})"
+            st.write(msg)
+            st.write("Score breakdown (1-4):")
+            st.dataframe(eval_df)
+
+#########################################
+    # examples section
+    st.subheader("")
+    st.markdown("***")
+    st.subheader("")
+
+    # # generate examples dropdown
+    # # st.subheader("Here are some example prompts and responses for you to try:")
+    # examples = []
+    # selected_example = st.selectbox('Select an example prompt:',examples)
+    #
+    # # default minimum and maximum scores for each example
+    # mins_ex = [2,1,0,0,0,0,0,0]
+    # maxes_ex = [12,6,3,3,4,4,30,60]
+    #
+    # # widgets and button to run on examples
+    # prompt_ta_ex = st.text_area("Essay Prompt:",placeholder="Input the question or the essay prompt here.", value=selected_example,key='prompt_ta_ex',height=175)
+    # response_ta_ex = st.text_area("Essay Response:",placeholder="Input the answer / essay response here.",key='response_ta_ex',height=250)
+    # col1_ex, col2_ex, col3_ex = st.columns(3)
+    # selected_min = mins_ex[examples.index(selected_example)]
+    # selected_max = maxes_ex[examples.index(selected_example)]
+    # min_score_ex = col1_ex.number_input('Minimum Score',0,100,key='min_score_ex',value=selected_min)
+    # max_score_ex = col2_ex.number_input('Maximum Score',0,100,key='max_score_ex',value=selected_max)
+    # passing_rate_ex = col3_ex.number_input('Passing Rate',0.0,1.0,0.6,step=0.05,key='passing_rate_ex')
+    # run_button_ex = st.button("Grade Example Essay")
+    #
+    # # button is clicked
+    # if run_button_ex:
+    #     if not prompt_ta_ex or not response_ta_ex: # if any text area is empty:
+    #         st.error("Please input the prompt / response in their corresponding text area.")
+    #     else: # run model
+    #         score, passed = run_model(answer=response_ta_ex, min_score=min_score_ex, max_score=max_score_ex, passing_rate=passing_rate_ex)
+    #         # output message template
+    #         msg_ex = f"Your essay score is: {int(score)} (Minimum Possible Score: {min_score_ex} | Maximum Possible Score: {max_score_ex})"
+    #
+    #         if passed: # check if score achieved passes the threshold
+    #             st.success(f"Congratulations! {msg_ex}")
+    #         if not passed:
+    #             st.warning(msg_ex)
+
+
+if __name__ == '__main__':
+    main()
+
+# To run streamlit, go to terminal and type: 'streamlit run app-source.py'

data_reader.py

@@ -0,0 +1,40 @@
+import os
+from pathlib import Path
+from typing import Tuple
+
+import pandas as pd
+# from dotenv import load_dotenv
+#
+# load_dotenv()
+
+
+# def set_env_if_kaggle_environ() -> None:
+#     if 'KAGGLE_DATA_PROXY_TOKEN' in os.environ:
+#         os.environ['DATA_PATH'] = '/kaggle/input/feedback-prize-english-language-learning/'
+
+
+def load_train_test_df(is_testing: bool = False) -> Tuple[pd.DataFrame, pd.DataFrame]:
+    """Loads train/test dataframes
+    :param is_testing: If set to true, load subsample of train/test dataframes
+    :return Train and test dataframes
+    """
+    # set_env_if_kaggle_environ()
+
+    if is_testing:
+        train_df_path = Path("tests/data/train_sample.csv")
+        test_df_path = Path("tests/data/test_sample.csv")
+
+    else:
+        train_df_path = Path(os.environ['DATA_PATH']) / 'train.csv'
+        test_df_path = Path(os.environ['DATA_PATH']) / 'test.csv'
+
+    if not test_df_path.is_file():
+        raise OSError(f"File not found: {test_df_path.absolute()}")
+
+    if not train_df_path.is_file():
+        raise OSError(f"File not found: {train_df_path.absolute()}")
+
+    train_df = pd.read_csv(train_df_path)
+    test_df = pd.read_csv(test_df_path)
+
+    return train_df, test_df

losses.py

@@ -0,0 +1,28 @@
+import torch
+import torch.nn as nn
+from torch import Tensor
+
+
+class MCRMSELoss(nn.Module):
+
+    def __init__(self):
+        super(MCRMSELoss, self).__init__()
+        self.mse = nn.MSELoss(reduction='none')
+
+    def forward(self, y_pred: Tensor, y_true: Tensor):
+        """Calculate mean column-wise rmse on columns
+        :param y_pred: tensor of shape (bs, 6)
+        :param y_true: tensor of shape (bs, 6)
+        :return: tensor of shape 0 (scalar with grad)
+        """
+
+        mse = self.mse(y_pred, y_true).mean(0)  # column-wise mean
+        rmse = torch.sqrt(mse + 1e-7)
+
+        return rmse.mean()
+
+    def class_mcrmse(self, y_pred: Tensor, y_true: Tensor):
+        mse = self.mse(y_pred, y_true).mean(0)  # column-wise mean
+        rmse = torch.sqrt(mse + 1e-7)
+
+        return rmse.squeeze()

model.py

@@ -0,0 +1,134 @@
+"""
+    Copy pasted model from https://www.kaggle.com/code/yasufuminakama/fb3-deberta-v3-base-baseline-train/notebook
+"""
+import pytorch_lightning as pl
+import torch
+import torch.nn as nn
+from torch.optim import AdamW
+from torch.optim.lr_scheduler import CosineAnnealingLR
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+
+from data_reader import load_train_test_df
+from losses import MCRMSELoss
+
+
+def num_train_samples():
+    train_df, _ = load_train_test_df()
+    return len(train_df)
+
+
+class MeanPooling(nn.Module):
+    # taking mean of last hidden state with mask
+
+    def forward(self, last_hidden_state, attention_mask):
+        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()
+        sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)
+        sum_mask = input_mask_expanded.sum(1)
+        sum_mask = torch.clamp(sum_mask, min=1e-9)
+        mean_embeddings = sum_embeddings / sum_mask
+        return mean_embeddings
+
+
+class BertLightningModel(pl.LightningModule):
+
+    def __init__(self, config: dict):
+        super(BertLightningModel, self).__init__()
+
+        self.config = config
+
+        huggingface_config = AutoConfig.from_pretrained(self.config['model_name'], output_hidden_states=True)
+        huggingface_config.hidden_dropout = 0.
+        huggingface_config.hidden_dropout_prob = 0.
+        huggingface_config.attention_dropout = 0.
+        huggingface_config.attention_probs_dropout_prob = 0.
+
+        self.tokenizer = AutoTokenizer.from_pretrained(self.config['model_name'])
+        self.model = AutoModel.from_pretrained(self.config['model_name'], config=huggingface_config)
+
+        self.pool = MeanPooling()
+
+        self.fc = nn.Linear(in_features=1024, out_features=6)
+
+        self.loss = MCRMSELoss()
+
+        # freezing first 20 layers of DeBERTa from 24
+        modules = [self.model.embeddings, self.model.encoder.layer[:self.config['num_frozen_layers']]]
+        for module in modules:
+            for param in module.parameters():
+                param.requires_grad = False
+
+        self.class_metric = None
+        self.best_metric = None
+
+    def forward(self, inputs):
+        outputs = self.model(**inputs)
+        last_hidden_state = outputs.last_hidden_state
+
+        bert_features = self.pool(last_hidden_state, inputs['attention_mask'])
+
+        logits = self.fc(bert_features)
+
+        return logits
+
+    def training_step(self, batch, batch_idx):
+        inputs = batch
+        labels = inputs.pop("labels", None)
+        logits = self(inputs)
+        loss = self.loss(logits, labels)
+
+        self.log('train/loss', loss)
+
+        return {
+            'loss': loss,
+            'mc_rmse': loss
+        }
+
+    def training_epoch_end(self, outputs):
+        mean_mc_rmse = sum(output['mc_rmse'].item() for output in outputs) / len(outputs)
+        self.log("train/epoch_loss", mean_mc_rmse)
+
+    def validation_step(self, batch, batch_idx):
+        inputs = batch
+        labels = inputs.pop("labels", None)
+        logits = self(inputs)
+        loss = self.loss(logits, labels)
+        class_rmse = self.loss.class_mcrmse(logits, labels)
+
+        self.log('val/loss', loss)
+
+        return {
+            'loss': loss,
+            'mc_rmse': loss,
+            'class_mc_rmse': class_rmse
+        }
+
+    def validation_epoch_end(self, outputs):
+        mean_mc_rmse = sum(output['mc_rmse'].item() for output in outputs) / len(outputs)
+        class_metrics = torch.stack([output['class_mc_rmse'] for output in outputs]).mean(0).tolist()
+        class_metrics = [round(item, 4) for item in class_metrics]
+        self.log('val/epoch_loss', mean_mc_rmse)
+
+        if self.best_metric is None or mean_mc_rmse < self.best_metric:
+            self.best_metric = mean_mc_rmse
+            self.class_metric = class_metrics
+
+    def configure_optimizers(self):
+        # weight_decay = self.config['weight_decay']
+        lr = self.config['lr']
+
+        # In original solution authors add weight decaying to some parameters
+
+        optimizer = AdamW(self.parameters(), lr=lr, weight_decay=0.0, eps=1e-6, betas=(0.9, 0.999))
+
+        scheduler = CosineAnnealingLR(
+            optimizer,
+            T_max=self.config['max_epochs'],
+        )
+        return [optimizer], [scheduler]
+
+    def predict_step(self, batch, batch_idx: int, dataloader_idx: int = 0):
+        inputs = batch
+        inputs.pop("labels", None)
+        logits = self(inputs)
+
+        return logits

requirements.txt

@@ -0,0 +1,10 @@
+streamlit==1.16.0
+numpy==1.21.6
+pandas==1.3.5
+torch==1.12.1
+transformers==4.22.2
+scikit-learn==1.0.2
+nltk==3.7
+seaborn==0.12.1
+pytorch-lightning==1.8.3
+sentencepiece==0.1.97

tf_model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c22d4ab171b1e8594cac4e4f8030010d0fcad9c69caf5c7036a4f6650d82330
+size 1736236975