Spaces:
Runtime error
Runtime error
| """ | |
| This module contains methods for extracting text sentiment from texts | |
| """ | |
| import torch | |
| import pandas as pd | |
| import numpy as np | |
| from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer | |
| # ref: https://colab.research.google.com/github/chrsiebert/sentiment-roberta-large-english/blob/main/sentiment_roberta_prediction_example.ipynb | |
| # Create class for data preparation | |
| class SimpleDataset: | |
| def __init__(self, tokenized_texts): | |
| self.tokenized_texts = tokenized_texts | |
| def __len__(self): | |
| return len(self.tokenized_texts["input_ids"]) | |
| def __getitem__(self, idx): | |
| return {k: v[idx] for k, v in self.tokenized_texts.items()} | |
| class Sentiment_Extractor: | |
| def __init__(self,input_file_name,text_column,output_file_name): | |
| self.input_file_name = input_file_name | |
| self.text_column = text_column | |
| self.output_file_name = output_file_name | |
| def run(self): | |
| # Load tokenizer and model, create trainer | |
| model_name = "siebert/sentiment-roberta-large-english" | |
| tokenizer = AutoTokenizer.from_pretrained(model_name) | |
| model = AutoModelForSequenceClassification.from_pretrained(model_name) | |
| trainer = Trainer(model=model) | |
| df_pred = pd.read_csv(self.input_file_name,encoding='cp1255') | |
| pred_texts = df_pred[self.text_column].dropna().astype('str').tolist() | |
| # Tokenize texts and create prediction data set | |
| tokenized_texts = tokenizer(pred_texts,truncation=True,padding=True) | |
| pred_dataset = SimpleDataset(tokenized_texts) | |
| # Run predictions | |
| predictions = trainer.predict(pred_dataset) | |
| # Transform predictions to labels | |
| preds = predictions.predictions.argmax(-1) | |
| labels = pd.Series(preds).map(model.config.id2label) | |
| scores = (np.exp(predictions[0])/np.exp(predictions[0]).sum(-1,keepdims=True)).max(1) | |
| # Create DataFrame with texts, predictions, labels, and scores | |
| df = pd.DataFrame(list(zip(pred_texts,preds,labels,scores)), columns=['text_sentiment','pred_sentiment','label_sentiment','score_sentiment']) | |
| df_output = df_pred.merge(df,left_on=self.text_column,right_on='text_sentiment') | |
| del df_output['text_sentiment'] | |
| df_output.to_csv(self.output_file_name,encoding='cp1255',index=False) | |
| if __name__ == "__main__": | |
| # Arguments | |
| # INPUT_FILE_NAME is the name of the input file | |
| INPUT_FILE_NAME = "tagging_MMD_db_with_summarized.csv" | |
| # TEXT_COLUMN is the name of the text column in the input file | |
| # from which we extract the positive / negative sentiment by the 🤗 model. | |
| TEXT_COLUMN = "text" | |
| OUTPUT_FILE_NAME = 'tagging_MMD_db_with_sentiment.csv' | |
| # Run Sentiment_Extractor on the given arguments | |
| obj = Sentiment_Extractor(INPUT_FILE_NAME,OUTPUT_FILE_NAME) | |
| obj.run() |