Spaces:
Sleeping
Sleeping
| import pandas as pd | |
| import numpy as np | |
| from torch.utils.data import Dataset | |
| import torch | |
| from transformers import AutoTokenizer | |
| import pytorch_lightning as pl | |
| from torch.utils.data import DataLoader | |
| from transformers import AutoModel, AdamW, get_cosine_schedule_with_warmup | |
| import torch.nn as nn | |
| import math | |
| from torchmetrics.functional.classification import auroc | |
| import torch.nn.functional as F | |
| import streamlit as st | |
| from transformers import pipeline | |
| class toxicity_dataset(Dataset): | |
| def __init__(self,data_path,tokenizer,attributes,max_token_len= 128,sample = 1000): | |
| self.data_path=data_path | |
| self.tokenizer=tokenizer | |
| self.attributes=attributes | |
| self.max_token_len=max_token_len | |
| self.sample=sample | |
| self._prepare_data() | |
| def _prepare_data(self): | |
| data=pd.read_csv(self.data_path) | |
| if self.sample is not None: | |
| self.data=data.sample(self.sample,random_state=7) | |
| else: | |
| self.data=data | |
| def __len__(self): | |
| return(len(self.data)) | |
| def __getitem__(self,index): | |
| item = self.data.iloc[index] | |
| comment = str(item.comment_text) | |
| attributes = torch.FloatTensor(item[self.attributes]) | |
| tokens = self.tokenizer.encode_plus(comment,add_special_tokens=True,return_tensors="pt",truncation=True,max_length=self.max_token_len,padding="max_length",return_attention_mask=True) | |
| return{'input_ids':tokens.input_ids.flatten(),"attention_mask":tokens.attention_mask.flatten(),"labels":attributes} | |
| class Toxcity_Data_Module(pl.LightningDataModule): | |
| def __init__(self,train_path,test_path,attributes,batch_size = 16, max_token_len = 128, model_name="roberta-base"): | |
| super().__init__() | |
| self.train_path=train_path | |
| self.test_path=test_path | |
| self.attributes=attributes | |
| self.batch_size=batch_size | |
| self.max_token_len=max_token_len | |
| self.model_name=model_name | |
| self.tokenizer = AutoTokenizer.from_pretrained(model_name) | |
| def setup(self, stage = None): | |
| if stage in (None, "fit"): | |
| self.train_dataset=toxicity_dataset(self.train_path,self.tokenizer,self.attributes) | |
| self.test_dataset=toxicity_dataset(self.test_path,self.tokenizer,self.attributes, sample=None) | |
| if stage == "predict": | |
| self.val_dataset=toxicity_dataset(self.test_path,self.tokenizer,self.attributes) | |
| def train_dataloader(self): | |
| return DataLoader(self.train_dataset,batch_size=self.batch_size,shuffle=True) | |
| def val_dataloader(self): | |
| return DataLoader(self.train_dataset,batch_size=self.batch_size,shuffle=False) | |
| def predict_dataloader(self): | |
| return DataLoader(self.test_dataset,batch_size=self.batch_size,shuffle=False) | |
| class Toxic_Comment_Classifier(pl.LightningModule): | |
| def __init__(self, config: dict): | |
| super().__init__() | |
| self.config = config | |
| self.pretrained_model = AutoModel.from_pretrained(config['model_name'], return_dict = True) | |
| self.hidden = torch.nn.Linear(self.pretrained_model.config.hidden_size, self.pretrained_model.config.hidden_size) | |
| self.classifier = torch.nn.Linear(self.pretrained_model.config.hidden_size, self.config['n_labels']) | |
| torch.nn.init.xavier_uniform_(self.classifier.weight) | |
| self.loss_func = nn.BCEWithLogitsLoss(reduction='mean') | |
| self.dropout = nn.Dropout() | |
| def forward(self, input_ids, attention_mask=None, labels=None): | |
| # roberta layer | |
| output = self.pretrained_model(input_ids=input_ids, attention_mask=attention_mask) | |
| pooled_output = torch.mean(output.last_hidden_state, 1) | |
| # final logits | |
| pooled_output = self.dropout(pooled_output) | |
| pooled_output = self.hidden(pooled_output) | |
| pooled_output = F.relu(pooled_output) | |
| pooled_output = self.dropout(pooled_output) | |
| logits = self.classifier(pooled_output) | |
| # calculate loss | |
| loss = 0 | |
| if labels is not None: | |
| loss = self.loss_func(logits.view(-1, self.config['n_labels']), labels.view(-1, self.config['n_labels'])) | |
| return loss, logits | |
| def training_step(self, batch, batch_index): | |
| loss, outputs = self(**batch) | |
| self.log("train loss ", loss, prog_bar = True, logger=True) | |
| return {"loss":loss, "predictions":outputs, "labels": batch["labels"]} | |
| def validation_step(self, batch, batch_index): | |
| loss, outputs = self(**batch) | |
| self.log("validation loss ", loss, prog_bar = True, logger=True) | |
| return {"val_loss": loss, "predictions":outputs, "labels": batch["labels"]} | |
| def predict_step(self, batch, batch_index): | |
| loss, outputs = self(**batch) | |
| return outputs | |
| def configure_optimizers(self): | |
| optimizer = AdamW(self.parameters(), lr=self.config['lr'], weight_decay=self.config['w_decay']) | |
| total_steps = self.config['train_size']/self.config['bs'] | |
| warmup_steps = math.floor(total_steps * self.config['warmup']) | |
| warmup_steps = math.floor(total_steps * self.config['warmup']) | |
| scheduler = get_cosine_schedule_with_warmup(optimizer, warmup_steps, total_steps) | |
| return [optimizer],[scheduler] | |
| def predict_raw_comments(model, dm, trainer): | |
| print("debug1") | |
| predictions = trainer.predict(model,dm) | |
| print("debug2") | |
| flattened_predictions = np.stack([torch.sigmoid(torch.Tensor(p)) for batch in predictions for p in batch]) | |
| print("debug3") | |
| return flattened_predictions | |
| def main(): | |
| # -- Creates Variables for Use of Model -- | |
| attributes=["toxic","severe_toxic","obscene","threat","insult","identity_hate"] | |
| tokenizer=AutoTokenizer.from_pretrained("roberta-base") | |
| toxic_comments_dataset=toxicity_dataset("data/train.csv",tokenizer,attributes) | |
| toxicity_data_module=Toxcity_Data_Module("data/train.csv","data/test.csv",attributes) | |
| toxicity_data_module.setup() | |
| dataloader=toxicity_data_module.train_dataloader() | |
| config = { | |
| 'model_name':"distilroberta-base", | |
| 'n_labels':len(attributes), | |
| 'bs':128, | |
| 'lr':1.5e-6, | |
| 'warmup':0.2, | |
| "train_size":len(toxicity_data_module.train_dataloader()), | |
| 'w_decay':0.001, | |
| 'n_epochs':1 | |
| } | |
| toxicity_data_module=Toxcity_Data_Module("data/train.csv","data/reduced_test.csv",attributes,batch_size=config['bs']) | |
| toxicity_data_module.setup() | |
| trainer = pl.Trainer(max_epochs=config['n_epochs'],num_sanity_val_steps=50) | |
| ## -- Creates Streamlit App -- | |
| st.title("Tweet Toxicity Classifier ") | |
| st.header("Fine tuned model from roberta-base using PyTorch") | |
| st.header("Jozef Janosko - CS 482, Milestone 3") | |
| model_name = st.selectbox("Select Model...", ["Toxicity Classification Model"]) | |
| if st.button("Click to Load Data"): | |
| if model_name=="Toxicity Classification Model": | |
| model = torch.load("ToxicityClassificationModel.pt") | |
| with st.spinner('Analyzing Text...'): | |
| logits = predict_raw_comments(model,toxicity_data_module,trainer=trainer) | |
| torch_logits = torch.from_numpy(logits) | |
| probabilities = F.softmax(torch_logits, dim = -1).numpy() | |
| inputs=pd.read_csv("data/reduced_test.csv") | |
| data=[] | |
| #print(inputs["comment_text"][0]," ",probabilities) | |
| for i in range(len(probabilities)): | |
| max_prob = 0 | |
| max_cat = 6 | |
| prob=0 | |
| for j in range(6): | |
| prob=probabilities[i][j] | |
| if(prob >= max_prob): | |
| max_prob = prob | |
| max_cat = j | |
| #print(inputs["comment_text"][i]," ",attributes[max_cat]," ",max_prob," ",probabilities[i]) | |
| data.append([inputs["comment_text"][i][0:16],attributes[max_cat],max_prob]) | |
| results_df=pd.DataFrame(data,columns=["Comment Text","Most Likely Classification","Classification Probability"]) | |
| st.table(data=results_df) | |
| else: | |
| model = pipeline("sentiment-analysis",model_name) | |
| if __name__ == '__main__' : | |
| main() |