Final assignment submission

.env

@@ -0,0 +1 @@
+OMDB_KEY=17c73f9a

app.py

@@ -0,0 +1,97 @@
+import numpy as np
+import gradio as gr
+from recommender import Recommender
+
+css="""
+.gradio-row {
+  flex-wrap: nowrap !important;
+}
+
+.btn {
+  height: 50px !important;
+  max-height: 50px !important
+}
+"""
+
+# Create recommender object
+recommender = Recommender()
+initial_picks = recommender.get_descs_for_recommended(recommender.recommended_ids)
+
+with gr.Blocks(css=css) as demo:
+    gr.Markdown("# **Full report and code can be found here: [GitHub](https://github.com/SSBakh07/Statistical-ML---Spring-2023)**")
+    gr.Markdown("## Basic Movie Recommender")
+    with gr.Row(variant='compact', elem_classes="gradio-row", equal_height=True):
+
+      # First Component
+      with gr.Column(scale=1):
+        col_1_number = gr.Number(value=1, visible=False)
+        
+        gr.Markdown("Based on similar movies...")
+
+        movie_title_1 = gr.Textbox(initial_picks[0]['title'], label="Movie Title")
+        movie_summary_1 = gr.Textbox(initial_picks[0]['overview'], label="Movie Summary")
+
+        
+        with gr.Column():
+          gr.Markdown("How much did you enjoy this movie or how interested are you?")
+          slider_1 = gr.Slider(minimum=1, maximum=5, editable=True)
+          btn_submit_1 = gr.Button("Submit", elem_classes="btn")
+
+
+
+      # Second Component
+      with gr.Column(scale=1):
+        col_2_number = gr.Number(value=2, visible=False)
+        
+        gr.Markdown("Based on similar users...")
+
+        movie_title_2 = gr.Textbox(initial_picks[1]['title'], label="Movie Title")
+        movie_summary_2 = gr.Textbox(initial_picks[1]['overview'], label="Movie Summary")
+
+        
+        with gr.Column():
+          gr.Markdown("How much did you enjoy this movie or how interested are you?")
+          slider_2 = gr.Slider(minimum=1, maximum=5, editable=True)
+          btn_submit_2 = gr.Button("Submit", elem_classes="btn")
+
+
+
+      # Third Component
+      with gr.Column(scale=1):
+        col_3_number = gr.Number(value=3, visible=False)
+
+        gr.Markdown("Based on similar users and movies...")
+
+        movie_title_3 = gr.Textbox(initial_picks[2]['title'], label="Movie Title")
+        movie_summary_3 = gr.Textbox(initial_picks[2]['overview'], label="Movie Summary")
+
+
+        with gr.Column():
+          gr.Markdown("How much did you enjoy this movie or how interested are you?")
+          slider_3 = gr.Slider(minimum=1, maximum=5, editable=True)
+          btn_submit_3 = gr.Button("Submit", elem_classes="btn")
+
+
+
+    #Handler functions    
+    def submit_opinion(number, value):
+      global recommender
+      res = recommender.on_pick(int(number), value)
+      text_res = recommender.get_descs_for_recommended(res)
+      final = []
+      for txt in text_res:
+        final.append(txt['title'])
+        final.append(txt['overview'])
+      return final
+    
+    
+    # Attach buttons to functions
+    submit_outputs = [movie_title_1, movie_summary_1, movie_title_2, 
+                      movie_summary_2, movie_title_3, movie_summary_3]
+    btn_submit_1.click(submit_opinion, [col_1_number, slider_1], submit_outputs)
+    btn_submit_2.click(submit_opinion, [col_2_number, slider_2], submit_outputs)
+    btn_submit_3.click(submit_opinion, [col_3_number, slider_3], submit_outputs)
+    
+
+
+demo.launch(debug=True)

items_0.csv

@@ -0,0 +1,2 @@
+adult,id,imdb_id,overview,popularity,runtime,title,vote_average,Animation,Comedy,Family,Adventure,Fantasy,Romance,Drama,Action,Crime,Thriller,Horror,History,Science Fiction,Mystery,War,Foreign,Music,Documentary,Western,TV Movie,rb_ratio,pop_bin
+True,862,tt0114709,"Led by Woody, Andy's toys live happily in his room until Andy's birthday brings Buzz Lightyear onto the scene. Afraid of losing his place in Andy's heart, Woody plots against Buzz. But when circumstances separate Buzz and Woody from their owner, the duo eventually learns to put aside their differences.",21.946943,81.0,Toy Story,7.7,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,12.4518011,9

items_1.csv

@@ -0,0 +1,2 @@
+adult,id,imdb_id,overview,popularity,runtime,title,vote_average,Animation,Comedy,Family,Adventure,Fantasy,Romance,Drama,Action,Crime,Thriller,Horror,History,Science Fiction,Mystery,War,Foreign,Music,Documentary,Western,TV Movie,rb_ratio,pop_bin
+True,27678,tt0106356,"A television movie based upon the book by Bryan Burrough and John Helyar, about the leveraged buyout (LBO) of RJR Nabisco.",1.685697,107.0,Barbarians at the Gate,6.8,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0.3026610976581805,5

items_2.csv

@@ -0,0 +1,2 @@
+adult,id,imdb_id,overview,popularity,runtime,title,vote_average,Animation,Comedy,Family,Adventure,Fantasy,Romance,Drama,Action,Crime,Thriller,Horror,History,Science Fiction,Mystery,War,Foreign,Music,Documentary,Western,TV Movie,rb_ratio,pop_bin
+True,18520,tt0799916,"Join filmmaking duo Chris Hegedus and Nick Doob as their cameras follow Franken to book signings, campaign rallies and the launch of Air America Radio, documenting his transformation from irreverent funnyman to political pundit.",0.364839,84.0,Al Franken - God Spoke,6.8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0.018583582460259,2

items_3.csv

@@ -0,0 +1,2 @@
+adult,id,imdb_id,overview,popularity,runtime,title,vote_average,Animation,Comedy,Family,Adventure,Fantasy,Romance,Drama,Action,Crime,Thriller,Horror,History,Science Fiction,Mystery,War,Foreign,Music,Documentary,Western,TV Movie,rb_ratio,pop_bin
+True,51548,tt1313254,"Spying on her students, a teacher finds an exciting hobby.",0.588512,97.0,The Invisible Eye,6.2,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0.0689346370279518,3

items_4.csv

@@ -0,0 +1,2 @@
+adult,id,imdb_id,overview,popularity,runtime,title,vote_average,Animation,Comedy,Family,Adventure,Fantasy,Romance,Drama,Action,Crime,Thriller,Horror,History,Science Fiction,Mystery,War,Foreign,Music,Documentary,Western,TV Movie,rb_ratio,pop_bin
+True,164443,tt2377132,"This is an update of George Bernard Shaw's ""Pygmalion"" that changes the genders of the main characters. Hannah Higgins attempts to turn blue-collar Boston beer vendor Elliot Doolittle into a viable candidate and inadvertently learns something of Elliot's side of life.",5.152674,85.0,The Makeover,5.5,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2.0673618764144663,8

recommender.py

@@ -0,0 +1,236 @@
+# Supress sklearn warnings
+def warn(*args, **kwargs):
+    pass
+import warnings
+warnings.warn = warn
+
+# Import libraries
+import numpy as np
+import pandas as pd
+from glob import glob
+from sklearn.neighbors import NearestNeighbors 
+from sklearn.preprocessing import MinMaxScaler
+
+# The columns that will be taken into account when making item-based similarity recommendations
+item_columns = []    
+
+# Number of neighbors to take into account
+N_NEIGHBORS = 10
+
+
+# Handler for Item DataFrame
+class ItemData:
+    def __init__(self):
+        self.df = pd.concat([pd.read_csv(f) for f in glob("items_*.csv")], axis=0)
+        self._scale_cols()
+        self.item_columns = ['scaled_runtime', 'vote_scaled', 'Animation', 'Comedy', 'Family', 'Adventure', 'Fantasy', 'Romance', 'Drama', 'Action', 
+            'Crime', 'Thriller', 'Horror', 'History','Science Fiction', 'Mystery', 'War', 'Foreign', 'Music', 'Documentary', 
+            'Western', 'TV Movie', 'ratio_scaled', 'pop_scaled']
+        self.scaled_df = self.df[self.item_columns]
+
+    def _scale_cols(self):
+        runtime_col = self.df['runtime'].values.reshape(-1, 1)
+        runtime_scaler = MinMaxScaler().fit(runtime_col)
+        self.df['scaled_runtime'] = runtime_scaler.transform(runtime_col)
+
+        vote_col = self.df['vote_average'].values.reshape(-1, 1)
+        vote_scaler = MinMaxScaler().fit(vote_col)
+        self.df['vote_scaled'] = vote_scaler.transform(vote_col)
+
+        ratio_col = self.df['rb_ratio'].values.reshape(-1, 1)
+        ratio_scaler = MinMaxScaler().fit(ratio_col)
+        self.df['ratio_scaled'] = ratio_scaler.transform(ratio_col)
+
+        pop_col = self.df['pop_bin'].values.reshape(-1, 1)
+        ratio_scaler = MinMaxScaler().fit(pop_col)
+        self.df['pop_scaled'] = ratio_scaler.transform(pop_col)
+    
+    def get_filtered_row_by_id(self, id):
+        return self.df[self.df['id'] == int(id)][self.item_columns]
+    
+    def get_id_by_idx(self, idx):
+        return self.df.at[idx, 'id']
+    
+    def get_random_id(self):
+        return self.df.sample(1)['id'].values[0]
+    
+    def get_row_by_id(self, id):
+        return self.df[self.df['id'] == id]
+
+    def get_movie_title_by_id(self, id):
+        return self.get_row_by_id(id)['title'].values[0]
+    
+    def get_movie_overview_by_id(self, id):
+        return self.get_row_by_id(id)['overview'].values[0]
+        
+
+# Handler for User DataFrame
+class UserData:
+    def __init__(self):
+        self.df = pd.concat([pd.read_csv(f) for f in glob("users_*.csv")], axis=0)
+        self.df = self.df.fillna(0)
+
+
+###### Recommender System
+class Recommender:
+    def __init__(self):
+        # Load preprocessed dataframes
+        self.item_handler = ItemData()
+        self.user_handler = UserData()
+        print("Dataframes loaded...")
+
+        self.preferences = pd.DataFrame(columns=self.user_handler.df.columns[1:])    # For user data
+        self.preferences.loc[0] = 0    # Initialize all ratings to zero
+
+        self.item_picks = pd.DataFrame(columns=self.item_handler.df.columns)
+        
+        self.n_picks = 1
+        self.recommended_ids = []    # Resets every time
+        self.seen_movies = []
+
+        # Initialize nearest neighbor algorithm. With p=1, euclidean distance is our metric
+        self.user_recommender = NearestNeighbors(n_neighbors=N_NEIGHBORS, p=2).fit(self.user_handler.df.drop('user_id', axis=1))
+        self.item_recommender = NearestNeighbors(n_neighbors=N_NEIGHBORS, p=2).fit(self.item_handler.scaled_df)
+
+        # Initialize recommended movies
+        for i in range(3):
+            self.recommended_ids.append(self.get_item_recommendation())    # Getting random movies
+
+
+    def on_pick(self, idx, rating):
+        '''
+            Called whenever the user picks a new movie.
+            idx: [0, 2] -> which one of the recommendations was picked out of the 3 suggestions
+        '''
+        self.n_picks += 1
+
+        chosen_movie_id = self.recommended_ids[idx]
+        self.update(chosen_movie_id, rating)
+
+        # Recommend new movies
+        self.recommended_ids[0] = self.get_item_recommendation()
+        self.recommended_ids[1] = self.get_user_recommendation()
+        self.recommended_ids[2] = self.get_joint_recommendation()
+
+        return self.recommended_ids
+    
+    def get_descs_for_recommended(self, recs):
+        descs = []
+        for rec in recs:
+            info = {}
+            info['title'] = self.item_handler.get_movie_title_by_id(rec)
+            info['overview'] = self.item_handler.get_movie_overview_by_id(rec)
+            descs.append(info)
+        return descs
+
+    def update(self, movie_id, rating):
+        '''
+            Update user preferences based on last picked movie (and given rating)
+        '''
+        self.seen_movies.append(movie_id)
+
+        # Update user data
+        self.preferences.at[0, str(movie_id)] = rating
+
+        # Update item data - but only if the user liked it
+        if rating > 2.5:
+            new_row = self.item_handler.get_row_by_id(movie_id)
+            self.item_picks = pd.concat([self.item_picks, new_row], axis=0)
+
+    
+    def get_item_recommendation(self):
+        '''
+            Make recommendation based on item similarity
+        '''
+        # If user hasn't picked any movies they like yet, pick something random
+        if not self.item_picks.empty:
+            filtered_picks = self.item_picks[self.item_handler.item_columns]
+            
+            # Return movie that's closest to average preference
+            summed_preferences = filtered_picks.sum(axis=0)
+            average_preferences = summed_preferences / filtered_picks.shape[0]
+
+            dist, idxes = self.item_recommender.kneighbors([average_preferences], min(len(self.seen_movies), self.item_handler.df.shape[0]))    # guarenteed to pick a movie that has not been seen before
+            
+            for idx in idxes[0]:
+                new_id = self.item_handler.get_id_by_idx(idx)
+                if new_id not in self.seen_movies:
+                    return new_id
+
+        # Pick a random movie if strategy did not work
+        return self.item_handler.get_random_id()     
+
+    
+    def get_user_recommendation(self):
+        '''
+            Make recommendation based on user similarity
+        '''
+        # If user hasn't chosen anything yet
+        if self.item_picks.empty:
+            return self.item_handler.get_random_id()
+
+        _, idx = self.user_recommender.kneighbors(self.preferences.values, 25)
+
+        # Find the closest user's top 3 movies. If all have been seen, move onto the next user until a candidate movie is found
+        for best_idx in idx[0]:
+            cols_to_drop = ['user_id']
+            # Find best movie
+            for i in range(3):
+                best_movie = self.user_handler.df.drop(cols_to_drop, axis=1).iloc[best_idx].idxmax(axis=0)
+                if best_movie in self.seen_movies:
+                    cols_to_drop.append(best_movie)
+                    continue
+                
+                if self.user_handler.df.at[best_idx, best_movie] > 2.5:
+                    return int(best_movie)
+
+        # Otherwise, return random movie
+        return self.item_handler.get_random_id()
+
+    
+    def get_joint_recommendation(self):
+        '''
+            Make recommendation based on both item and user similarity
+        '''
+        # If user hasn't chosen anything yet
+        if self.item_picks.empty:
+            return self.item_handler.get_random_id()
+
+        # Get similar users
+        _, user_idxs = self.user_recommender.kneighbors(self.preferences.values, 10)
+
+        # Get similar items
+        summed_preferences = self.item_picks[self.item_handler.item_columns].sum(axis=0)
+        average_preferences = summed_preferences / self.item_picks.shape[0]
+
+        n_movies = min(len(self.seen_movies), self.item_handler.df.shape[0])
+        _, item_idxs = self.item_recommender.kneighbors([average_preferences], n_movies)    # guarenteed to pick a movie that has not been seen before
+
+        score_sums = [0 for i in range(n_movies)]
+        n_votes = [0 for i in range(n_movies)]
+
+        # Sum ratings per movie
+        for i, movie_idx in enumerate(item_idxs[0]):
+            movie_id = self.item_handler.get_id_by_idx(movie_idx)
+            if movie_id in self.seen_movies:
+                continue
+
+            for user_id in user_idxs[0]:
+                score = self.user_handler.df.at[user_id, str(movie_id)]
+                if score != 0:
+                    score_sums[i] += score
+                    n_votes[i] += 1
+        
+
+        # Calculate per-movie score
+        final_score = []
+        for i, score in enumerate(score_sums):
+            if n_votes[i] > 0:
+                final_score.append(score/n_votes[i])
+            else:
+                final_score.append(-1)
+
+        # Find best score
+        best_score_idx = final_score.index(max(final_score))
+        best_movie_idx = item_idxs[0][best_score_idx]
+        return self.item_handler.get_id_by_idx(best_movie_idx)