Updated app.py

app.py

@@ -7,6 +7,54 @@ text = st.text_input('Enter text')
 
 # out = model()
 
+model = AutoModelForSequenceClassification.from_pretrained("./bert-finetuned-sem_eval-english/checkpoint-36819")
+# model.to('cuda')
+
 if st.button("Predict"):
-  st.write("Genre: ")
+    st.write("Genre: ")
+
+    preprocess = Preprocess()
+    clean_plot = preprocess.apply(text)
+    st.write(clean_plot)
+
+    utility = Utility()
+    # id2label, label2id, tokenizer, df = utility.tokenize(df, genres)
+    # xtrain, xval, ytrain, yval = utility.train_test_split(df, y)
+    # xtrain_input_ids = [np.asarray(xtrain[i]['input_ids']) for i in range(xtrain.shape[0])]
+    # xtrain_attention_mask = [np.asarray(xtrain[i]['attention_mask']) for i in range(xtrain.shape[0])]
+    # xval_input_ids = [np.asarray(xval[i]['input_ids']) for i in range(xval.shape[0])]
+    # xval_attention_mask = [np.asarray(xval[i]['attention_mask']) for i in range(xval.shape[0])]
+
+    # # create Pandas DataFrame
+    # input_ids_labels_df = pd.DataFrame({'input_ids': xtrain_input_ids, 'attention_mask': xtrain_attention_mask, 'labels': ytrain.tolist()})
+    # # define data set object
+    # TD = CustomTextDataset(torch.IntTensor(input_ids_labels_df['input_ids']), torch.IntTensor(input_ids_labels_df['attention_mask']),\
+    #                     torch.FloatTensor(input_ids_labels_df['labels']))
+    # input_ids_labels_val_df = pd.DataFrame({'input_ids': xval_input_ids, 'attention_mask': xval_attention_mask, 'labels': yval.tolist()})
+    # VD = CustomTextDataset(torch.IntTensor(input_ids_labels_val_df['input_ids']), torch.IntTensor(input_ids_labels_val_df['attention_mask']),\
+    #                 torch.FloatTensor(input_ids_labels_val_df['labels']))
+
+    # # trainer = Trainer(
+    # #     model,
+    # #     train_dataset=TD,
+    # #     eval_dataset=VD,
+    # #     tokenizer=tokenizer,
+    # #     compute_metrics=compute_metrics
+    # # )               
+    # # y_pred = trainer.predict(VD)
+    # # y_pred = model(input_ids, attention_mask)
+    # preds = torch.FloatTensor(y_pred[0])
+
+    # y_predictions = []
+    # predictions = []
+
+    # for pred in preds:
+    #     # apply sigmoid + threshold
+    #     sigmoid = torch.nn.Sigmoid()
+    #     probs = sigmoid(pred.squeeze().cpu())
+    #     prediction = np.zeros(probs.shape)
+    #     prediction[np.where(probs >= 0.5)] = 1
+    #     predictions.append(prediction)
+    # y_pred = predictions
+
   # st.write(out)

preprocess.py

@@ -2,76 +2,16 @@ import json
 import pickle
 import pandas as pd
 import nltk
-from matplotlib import pyplot as plt
-import seaborn as sns
 import regex as re
 from nltk.corpus import stopwords
-from sklearn.preprocessing import MultiLabelBinarizer
 
 class Preprocess:
-    df = None
     genres = None
     y = None
     
     def __init__(self) -> None:
-        self.df = pd.read_csv('movies_genre.csv')
         self.genres = []
 
-    def plot_freq_dist(self):
-        all_genres = sum(self.genres, [])
-        all_genres = nltk.FreqDist(all_genres) 
-
-        # create frequency dataframe
-        all_genres_df = pd.DataFrame({'Genres': list(all_genres.keys()), 
-                                    'Count': list(all_genres.values())})
-                                    
-        g = all_genres_df.nlargest(columns="Count", n = 50) 
-        plt.figure(figsize=(12,15)) 
-        ax = sns.barplot(data=g, x= "Count", y = "Genres") 
-        ax.set(xlabel = 'Count',ylabel= 'Genre') 
-        plt.show()
-
-    # def extract_genre_values(self):
-    #     # extract genres
-    #     for row in self.df['genres']: 
-    #         self.genres.append(list(json.loads(row.replace("\'", "\"")).values())) 
-
-    #     # add to dataframe  
-    #     self.df['genres'] = self.genres
-
-    def retain_top_freq_genres(self):
-        for (index, row) in enumerate(self.df['genres']): 
-            self.genres.append(json.loads(row.replace("\'", "\"")))
-            self.df.at[index, "genres"] = json.loads(row.replace("\'", "\""))
-        # create frequency dataframe
-        all_genres = sum(self.genres,[])
-        all_genres = nltk.FreqDist(all_genres) 
-        all_genres_df = pd.DataFrame({'Genres': list(all_genres.keys()), 
-                                    'Count': list(all_genres.values())})
-
-        # # considering only top 35 frequent genres
-        # g = all_genres_df.nlargest(columns="Count", n = 35) 
-        # g.head()
-        # top_genres = list(g['Genres'])
-
-        # Genres with freq > 1000
-        all_genres_df = all_genres_df[all_genres_df["Count"] >= 8000]
-        top_genres = list(all_genres_df['Genres'])
-
-        # Removing genres which are not important
-        # top_genres.remove('Other')
-        # top_genres.remove('Crime Thriller')
-        # top_genres.remove('Movie')
-        # top_genres.remove('History')
-        # top_genres.remove('Bollywood')
-
-        # Removing genres other than top selected genres
-        for (index,row) in enumerate(self.df['genres']): 
-            row = [genre for genre in row if genre in top_genres]
-            self.df.at[index, "genres"] = row
-        
-        return top_genres
-
     def clean_text(self, text):
         """Cleans text by removing certains unwanted characters"""
 
@@ -92,35 +32,10 @@ class Preprocess:
         no_stopword_text = [w for w in text.split() if not w in stop_words]
         return ' '.join(no_stopword_text)
 
+    def apply(self, plot):
+        clean_plot = self.clean_text(str(plot))
 
-    def multi_label_binarizer(self):
-        multilabel_binarizer = MultiLabelBinarizer()
-        multilabel_binarizer.fit(self.df['genres'])
-
-        pickle.dump(multilabel_binarizer, open("models/multilabel_binarizer", 'wb'))
-
-        # transform target variable
-        self.y = multilabel_binarizer.transform(self.df['genres'])
-
-
-    def apply(self):
-        # remove samples with no plot
-        self.df = self.df[~(pd.isna(self.df['plot']))]
-        # self.df = self.df.head(20000)
-        # removing rows which has very small plot fewer than 500 characters
-        self.df = self.df[self.df["plot"].map(len) >= 500]
-        self.df = self.df.reset_index()
-
-        # self.extract_genre_values()
-
-        genres = self.retain_top_freq_genres()
-
-
-        self.df['clean_plot'] = self.df['plot'].apply(lambda x: self.clean_text(str(x)))
-
-        self.df['clean_plot'] = self.df['clean_plot'].apply(lambda x: self.remove_stopwords(str(x)))
-
-        self.multi_label_binarizer()
+        clean_plot = self.remove_stopwords(str(clean_plot))
 
-        return [self.df, self.y, genres]
+        return clean_plot
     

utility.py

@@ -1,54 +1,20 @@
 import pickle
 import wikipedia
 import numpy as np
-from sklearn.model_selection import train_test_split
-from skmultilearn.model_selection import iterative_train_test_split
-from sklearn.feature_extraction.text import TfidfVectorizer
 from transformers import AutoTokenizer
 
 class Utility:
     def __init__(self) -> None:
         pass
 
-    def get_summary(self,url):
-        summary = ""
-        try:
-            title = url.split("wiki/")[-1]
-            print(title)
-
-            wiki = wikipedia.page(title=title)
-
-            summary = wiki.summary
-        except:
-            pass
-
-        return summary
-
-
-    def get_plot(self,url):
-        plot=""
-
-        try:
-            title = url.split("wiki/")[-1]
-            wiki = wikipedia.page(title=title)
-
-            content = wiki.content.split('== Plot ==\n')[1]
-            plot = content.split('==')[0]
-        except:
-            pass
-
-        return plot
-
-    def tokenize(self, df, genres):
+    def tokenize(self, plot, genres):
         id2label = {idx:label for idx, label in enumerate(genres)}
         label2id = {label:idx for idx, label in enumerate(genres)}
 
         tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
 
-        df['clean_plot_tokenized'] = ''
-        for (idx, row) in df.iterrows():
-            df.at[idx,"clean_plot_tokenized"] = tokenizer(row["clean_plot"], padding="max_length", truncation=True, max_length=512)
-        return (id2label, label2id, tokenizer, df)
+        clean_plot_tokenized = tokenizer(plot, padding="max_length", truncation=True, max_length=512)
+        return (id2label, label2id, tokenizer, clean_plot_tokenized)
     
     def train_test_split(self, df, y):
         """Splits the dataset into training and validation set"""
@@ -61,14 +27,3 @@ class Utility:
         xval = np.array(xval).flatten()
 
         return (xtrain, xval, ytrain, yval)
-
-    
-    def vectorize(self, xtrain, xval):
-        """Creates TF-IDF features"""
-        tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=10000)
-        xtrain_tfidf = tfidf_vectorizer.fit_transform(xtrain)
-        xval_tfidf = tfidf_vectorizer.transform(xval)
-
-        pickle.dump(tfidf_vectorizer, open("models/tfidf_vectorizer", 'wb'))
-
-        return (xtrain_tfidf, xval_tfidf)