Added Utility and preprocess files

preprocess.py

@@ -0,0 +1,126 @@
+import json
+import pickle
+import pandas as pd
+import nltk
+from matplotlib import pyplot as plt
+import seaborn as sns
+import regex as re
+from nltk.corpus import stopwords
+from sklearn.preprocessing import MultiLabelBinarizer
+
+class Preprocess:
+    df = None
+    genres = None
+    y = None
+    
+    def __init__(self) -> None:
+        self.df = pd.read_csv('movies_genre.csv')
+        self.genres = []
+
+    def plot_freq_dist(self):
+        all_genres = sum(self.genres, [])
+        all_genres = nltk.FreqDist(all_genres) 
+
+        # create frequency dataframe
+        all_genres_df = pd.DataFrame({'Genres': list(all_genres.keys()), 
+                                    'Count': list(all_genres.values())})
+                                    
+        g = all_genres_df.nlargest(columns="Count", n = 50) 
+        plt.figure(figsize=(12,15)) 
+        ax = sns.barplot(data=g, x= "Count", y = "Genres") 
+        ax.set(xlabel = 'Count',ylabel= 'Genre') 
+        plt.show()
+
+    # def extract_genre_values(self):
+    #     # extract genres
+    #     for row in self.df['genres']: 
+    #         self.genres.append(list(json.loads(row.replace("\'", "\"")).values())) 
+
+    #     # add to dataframe  
+    #     self.df['genres'] = self.genres
+
+    def retain_top_freq_genres(self):
+        for (index, row) in enumerate(self.df['genres']): 
+            self.genres.append(json.loads(row.replace("\'", "\"")))
+            self.df.at[index, "genres"] = json.loads(row.replace("\'", "\""))
+        # create frequency dataframe
+        all_genres = sum(self.genres,[])
+        all_genres = nltk.FreqDist(all_genres) 
+        all_genres_df = pd.DataFrame({'Genres': list(all_genres.keys()), 
+                                    'Count': list(all_genres.values())})
+
+        # # considering only top 35 frequent genres
+        # g = all_genres_df.nlargest(columns="Count", n = 35) 
+        # g.head()
+        # top_genres = list(g['Genres'])
+
+        # Genres with freq > 1000
+        all_genres_df = all_genres_df[all_genres_df["Count"] >= 8000]
+        top_genres = list(all_genres_df['Genres'])
+
+        # Removing genres which are not important
+        # top_genres.remove('Other')
+        # top_genres.remove('Crime Thriller')
+        # top_genres.remove('Movie')
+        # top_genres.remove('History')
+        # top_genres.remove('Bollywood')
+
+        # Removing genres other than top selected genres
+        for (index,row) in enumerate(self.df['genres']): 
+            row = [genre for genre in row if genre in top_genres]
+            self.df.at[index, "genres"] = row
+        
+        return top_genres
+
+    def clean_text(self, text):
+        """Cleans text by removing certains unwanted characters"""
+
+        # remove backslash-apostrophe 
+        text = re.sub("\'", "", text) 
+        # remove everything except alphabets 
+        text = re.sub("[^a-zA-Z]"," ",text) 
+        # remove whitespaces 
+        text = ' '.join(text.split()) 
+        # convert text to lowercase 
+        text = text.lower() 
+
+        return text
+
+    def remove_stopwords(self,text):
+        """Function to remove stopwords""" 
+        stop_words = set(stopwords.words('english'))
+        no_stopword_text = [w for w in text.split() if not w in stop_words]
+        return ' '.join(no_stopword_text)
+
+
+    def multi_label_binarizer(self):
+        multilabel_binarizer = MultiLabelBinarizer()
+        multilabel_binarizer.fit(self.df['genres'])
+
+        pickle.dump(multilabel_binarizer, open("models/multilabel_binarizer", 'wb'))
+
+        # transform target variable
+        self.y = multilabel_binarizer.transform(self.df['genres'])
+
+
+    def apply(self):
+        # remove samples with no plot
+        self.df = self.df[~(pd.isna(self.df['plot']))]
+        # self.df = self.df.head(20000)
+        # removing rows which has very small plot fewer than 500 characters
+        self.df = self.df[self.df["plot"].map(len) >= 500]
+        self.df = self.df.reset_index()
+
+        # self.extract_genre_values()
+
+        genres = self.retain_top_freq_genres()
+
+
+        self.df['clean_plot'] = self.df['plot'].apply(lambda x: self.clean_text(str(x)))
+
+        self.df['clean_plot'] = self.df['clean_plot'].apply(lambda x: self.remove_stopwords(str(x)))
+
+        self.multi_label_binarizer()
+
+        return [self.df, self.y, genres]
+    

utility.py

@@ -0,0 +1,74 @@
+import pickle
+import wikipedia
+import numpy as np
+from sklearn.model_selection import train_test_split
+from skmultilearn.model_selection import iterative_train_test_split
+from sklearn.feature_extraction.text import TfidfVectorizer
+from transformers import AutoTokenizer
+
+class Utility:
+    def __init__(self) -> None:
+        pass
+
+    def get_summary(self,url):
+        summary = ""
+        try:
+            title = url.split("wiki/")[-1]
+            print(title)
+
+            wiki = wikipedia.page(title=title)
+
+            summary = wiki.summary
+        except:
+            pass
+
+        return summary
+
+
+    def get_plot(self,url):
+        plot=""
+
+        try:
+            title = url.split("wiki/")[-1]
+            wiki = wikipedia.page(title=title)
+
+            content = wiki.content.split('== Plot ==\n')[1]
+            plot = content.split('==')[0]
+        except:
+            pass
+
+        return plot
+
+    def tokenize(self, df, genres):
+        id2label = {idx:label for idx, label in enumerate(genres)}
+        label2id = {label:idx for idx, label in enumerate(genres)}
+
+        tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
+
+        df['clean_plot_tokenized'] = ''
+        for (idx, row) in df.iterrows():
+            df.at[idx,"clean_plot_tokenized"] = tokenizer(row["clean_plot"], padding="max_length", truncation=True, max_length=512)
+        return (id2label, label2id, tokenizer, df)
+    
+    def train_test_split(self, df, y):
+        """Splits the dataset into training and validation set"""
+        cleaned_plot_df = df['clean_plot_tokenized']
+        # xtrain, xval, ytrain, yval = train_test_split(cleaned_plot_df, y, test_size=0.2, random_state=9)
+
+        # stratified sampling
+        xtrain, ytrain, xval, yval = iterative_train_test_split(np.asmatrix(df['clean_plot_tokenized']).transpose(), y, test_size = 0.2)
+        xtrain = np.array(xtrain).flatten()
+        xval = np.array(xval).flatten()
+
+        return (xtrain, xval, ytrain, yval)
+
+    
+    def vectorize(self, xtrain, xval):
+        """Creates TF-IDF features"""
+        tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=10000)
+        xtrain_tfidf = tfidf_vectorizer.fit_transform(xtrain)
+        xval_tfidf = tfidf_vectorizer.transform(xval)
+
+        pickle.dump(tfidf_vectorizer, open("models/tfidf_vectorizer", 'wb'))
+
+        return (xtrain_tfidf, xval_tfidf)