# -*- coding: utf-8 -*- """ML Final Project Automatically generated by Colaboratory. Original file is located at https://colab.research.google.com/drive/1Aof3bcIIqSmvsh0cux6wZ5NPk1wY-l3D ### install dependencies """ !gdown "1W3-WEplVSztLR3lvkyYdiKZGMT4y0cNi&confirm=t" #!unzip IMDB.zip #!pip install mlflow """# Content-based filtering ### import libraries """ import numpy as np import pandas as pd import mlflow as mf #mf.log_artifacts({'rating':'/content/rating_small.csv', 'rating':'/content/rating_small.csv', 'movies':'/content/movies_metadata.csv','keywords':'/content/keywords.csv', 'credits':'/content/credits.csv'}) """### read data from file""" keywords = pd.read_csv('/content/IMDB/keywords.csv') keywords rating = pd.read_csv('/content/IMDB/ratings_small.csv') rating credits = pd.read_csv('/content/IMDB/credits.csv') credits metadata = pd.read_csv('/content/IMDB/movies_metadata.csv') metadata """keep only related columns from released movies:""" metadata = metadata[metadata['status'] == 'Released'] cols = np.array(['adult', 'belongs_to_collection', 'genres', 'id', 'original_language', 'title', 'production_countries', 'production_companies', 'video']) metadata = metadata[cols] metadata.iloc[1] def find_collection(x): if x == '': return '' return eval(str(x))['name'] metadata['belongs_to_collection'] = metadata['belongs_to_collection'].fillna('') metadata['belongs_to_collection'] = metadata['belongs_to_collection'].apply(find_collection) metadata.iloc[1] def find_names(x): if x == '': return '' genre_arr = eval(str(x)) return ','.join(i['name'] for i in eval(str(x))) metadata['genres'] = metadata['genres'].fillna('') metadata['genres']=metadata['genres'].apply(find_names) metadata['production_countries']=metadata['production_countries'].apply(find_names) metadata['production_companies']=metadata['production_companies'].apply(find_names) credits['cast'] = credits['cast'].apply(find_names) metadata.iloc[1] keywords['keywords'] = keywords['keywords'].apply(find_names) metadata['id'] = metadata['id'].astype(int) metadata = pd.merge(metadata,keywords,how='inner',on='id') metadata.iloc[1] def to_int(x): if x == 'True': return 1 return 0 metadata['adult'].unique() """there are 3 values other than True or False in adult column. there are entered by mistake so we remove those rows.""" metadata = metadata[(metadata['adult'] == 'True') | (metadata['adult'] == 'False')] metadata['adult'] = metadata['adult'].apply(to_int) metadata['video'].unique() """removing nan values from dataset and replacing 'True' and 'False' with 1 and 0:""" metadata = metadata[~metadata['video'].isna()] metadata['video'] = metadata['video'].apply(to_int) """## Vectorize string features""" metadata from sklearn.feature_extraction.text import CountVectorizer from sklearn.feature_extraction.text import TfidfVectorizer def my_tok(text): return text.split(",") def vectorize_string(col_name, feature_name, limit=None, df=metadata): vectorizer = CountVectorizer(tokenizer=my_tok, max_features=limit, min_df=2) X = vectorizer.fit_transform(df[col_name]) vec_cols = vectorizer.get_feature_names_out() vec_data = X.toarray() #vec_cols = np.char.add(feature_name+':', vec_cols) vec_cols = feature_name+':'+vec_cols return vec_data, vec_cols def tfidf(col_name, feature_name, limit=None, df=metadata): vectorizer = TfidfVectorizer(tokenizer=my_tok, max_features=limit, min_df=2) X = vectorizer.fit_transform(df[col_name]) vec_cols = vectorizer.get_feature_names_out() vec_data = X.toarray() #vec_cols = np.char.add(feature_name+':', vec_cols) vec_cols = feature_name+':'+vec_cols return vec_data, vec_cols genre_data, genre_cols = vectorize_string('genres', 'genre') genre_cols companies_data, companies_cols = vectorize_string('production_companies', 'company', 100) companies_cols countries_data, countries_cols = vectorize_string('production_countries', 'country') countries_cols collection_data, collection_cols = vectorize_string('belongs_to_collection', 'collection') collection_cols metadata['original_language']= metadata['original_language'].fillna('') lang_data, lang_cols = vectorize_string('original_language', 'lang') lang_cols collection_cols.shape keyword_data, keyword_cols = tfidf('keywords', 'keyword', 1000) keyword_cols credits.drop(columns=['crew'], inplace=True) credit_data, credit_cols = vectorize_string('cast','cast', 1000, df=credits) credit_cols metadata = pd.concat([metadata[['title','id','adult','video']], pd.DataFrame(genre_data, columns=genre_cols), pd.DataFrame(countries_data, columns=countries_cols), pd.DataFrame(collection_data, columns=collection_cols), pd.DataFrame(keyword_data, columns=keyword_cols), pd.DataFrame(companies_data, columns=companies_cols), pd.DataFrame(lang_data, columns=lang_cols)], axis=1) credits[credit_cols] = credit_data metadata = pd.merge(metadata, credits, how='inner', on='id') metadata #metadata.drop(['production_countries', 'genres', 'belongs_to_collection', 'keywords', 'production_companies', 'original_language'], axis=1, inplace=True) """list of all numerical features(everything except id and title)""" feature_cols = np.concatenate((np.array(['adult', 'video']), genre_cols,countries_cols,collection_cols,keyword_cols,companies_cols,lang_cols,credit_cols)) feature_cols #metadata[feature_cols] = metadata[feature_cols].astype('int8') del genre_data,countries_data,collection_data,keyword_data,companies_data,lang_data,credit_data del genre_cols,countries_cols,collection_cols,keyword_cols,companies_cols,lang_cols,credit_cols feature_cols.shape metadata def split_dataframe(df, holdout_fraction=0.1): test = df.sample(frac=holdout_fraction, replace=False) train = df[~df.index.isin(test.index)] return train, test train, test = split_dataframe(metadata) allIds = metadata['id'] number_of_batches = 4 batches = np.array_split(train, number_of_batches) mf.log_param('number of batches', number_of_batches) del metadata del train """## Algorithm """ batches[0] from sklearn.metrics.pairwise import cosine_similarity """`content_based_recommmeder` returns a list of movie ids based on it's input. the input should be a dataframe which has `movieId`, `rating` columns(like `ratings_small.csv` but without `userId`)""" number_of_batches =1 def content_based_recommender_movie(movieId): print("movie title is:", metadata[metadata['id']==movieId]) sim_mat= cosine_similarity(metadata[feature_cols]) return sim_mat #content_based_recommender_movie(272) batches[1].describe() from sklearn.metrics.pairwise import euclidean_distances as dist def content_based_recommender(user, df, k=10, movieIds=allIds): user_movies = pd.merge(user,df,how='inner',left_on='movieId',right_on='id') user_movies[feature_cols] = user_movies[feature_cols].multiply(user_movies['rating'], axis="index") mean_user_movies = user_movies[feature_cols].mean(axis=0) sim_mat = cosine_similarity(df[feature_cols][df.id.isin(movieIds)], mean_user_movies[feature_cols].values.reshape(1,-1)) temp_data = {'id':df['id'][df.id.isin(movieIds)], 'title':df['title'][df.id.isin(movieIds)], 'sim':sim_mat.flatten()} return pd.DataFrame(temp_data) def content_based_all_batches(user, k=10, movieIds=allIds): ans = content_based_recommender(user, batches[0], k, movieIds) for i in range(1,number_of_batches): ans.append(content_based_recommender(user, batches[i], k, movieIds)) return ans.sort_values(by='sim', ascending=False) content_based_k = 10 mf.log_param('content based k', content_based_k) #xx = content_based_recommender(rating[rating['userId'] == 1], batches[1], content_based_k) xx = content_based_all_batches(rating[rating['userId'] == 1], content_based_k) xx.shape """# Collaborative Filtering ### import libraries """ import numpy as np import pandas as pd from sklearn.utils.extmath import randomized_svd """### explore datasets""" rating = pd.read_csv('/content/IMDB/ratings_small.csv') rating.head() rating.shape links_small = pd.read_csv('/content/IMDB/links_small.csv') links_small.head() credits = pd.read_csv('/content/IMDB/credits.csv') credits.head() movie = pd.read_csv('/content/IMDB/movies_metadata.csv') movie.head() movie = movie.rename(columns={'id': 'movieId'}) movie.shape movie.head() """### data preprocessing There are three rows entered by mistake, so we remove that row. """ movie = movie[(movie['movieId']!='1997-08-20') & (movie['movieId']!='2012-09-29') & (movie['movieId']!='2014-01-01')] def find_names(x): if x == '': return '' genre_arr = eval(str(x)) return ','.join(i['name'] for i in eval(str(x))) movie['genres'] = movie['genres'].fillna('') movie['genres']=movie['genres'].apply(find_names) movie.movieId = movie.movieId.astype("uint64") """only keep rating for movies with metadata in movie dataset""" new_rating = pd.merge(rating, movie, how='inner', on=["movieId"]) new_rating = new_rating[["userId", "movieId", "rating"]] movie.head() new_rating.head() train, test = split_dataframe(new_rating) """### matrix factorization""" inter_mat_df = rating.pivot(index = 'userId', columns ='movieId', values = 'rating').fillna(0) inter_mat_df inter_mat = inter_mat_df.to_numpy() ratings_mean = np.mean(inter_mat, axis = 1) inter_mat_normal = inter_mat - ratings_mean.reshape(-1, 1) inter_mat_normal """We use singular value decomposition for matrix factorization""" svd_U, svd_sigma, svd_V = randomized_svd(inter_mat_normal, n_components=15, n_iter=5, random_state=47) """This function gives the diagonal form""" svd_sigma = np.diag(svd_sigma) """Making predictions""" rating_weights = np.dot(np.dot(svd_U, svd_sigma), svd_V) + ratings_mean.reshape(-1, 1) weights_df = pd.DataFrame(rating_weights, columns = inter_mat_df.columns) weights_df.head() """making recommendations""" def recommend_top_k(preds_df, ratings_df, movie, userId, k=10): user_row = userId-1 sorted_user_predictions = preds_df.iloc[user_row].sort_values(ascending=False) user_data = ratings_df[ratings_df.userId == (userId)] user_rated = user_data.merge(movie, how = 'left', left_on = 'movieId', right_on = 'movieId'). \ sort_values(['rating'], ascending=False) user_preds = movie.merge(pd.DataFrame(sorted_user_predictions).reset_index(), how = 'left', on = 'movieId').rename(columns = {user_row: 'prediction'}). \ sort_values('prediction', ascending = False). \ iloc[:k, :] return user_rated, user_preds collaborative_k = 100 user_rated, user_preds = recommend_top_k(weights_df, new_rating, movie, 220, collaborative_k) mf.log_param('collaborative k', collaborative_k) user_preds.head() user_rated.head() user_rated[["title", "genres"]].head(10) user_preds[["title", "genres"]].head(10) """# Ensemble Model""" def ensemble(userId, k=10): user_rated, user_preds = recommend_top_k(weights_df, new_rating, movie, userId, k*k) content_based_result = content_based_all_batches(rating[rating['userId'] == userId], k=k, movieIds=user_preds['movieId']) return content_based_result[['id','title']] ensemble_k=10 mf.log_param('ensemble k', ensemble_k) ensemble(220, ensemble_k) """# Evaluation""" df_res = user_preds[["movieId", "prediction"]]. \ merge(user_rated[["movieId", "rating"]], how = 'outer', on = 'movieId') df_res.sort_values(by='prediction',ascending=False,inplace=True) df_res threshold = 2 df_res['prediction'] = df_res['prediction'] >= threshold df_res['rating'] = df_res['rating'] >= threshold df_res def precision_at_k(df, k=10, y_test: str='rating', y_pred='prediction'): dfK = df.head(k) sum_df = dfK[y_pred].sum() true_pred = dfK[dfK[y_pred] & dfK[y_test]].shape[0] if sum_df > 0: return true_pred/sum_df else: return None def recall_at_k(df, k=10, y_test='rating', y_pred='prediction'): dfK = df.head(k) sum_df = df[y_test].sum() true_pred = dfK[dfK[y_pred] & dfK[y_test]].shape[0] if sum_df > 0: return true_pred/sum_df else: return None prec_at_k = precision_at_k(df_res, 100, y_test='rating', y_pred='prediction') rec_at_k = recall_at_k(df_res, 100, y_test='rating', y_pred='prediction') print("precision@k: ", prec_at_k) print("recall@k: ", rec_at_k) mf.log_metric('recall', rec_at_k) mf.log_metric('precision', prec_at_k) """# MLOps""" def updata_batch(new_batch): number_of_batches = number_of_batches+1 batches = batches.append(new_batch) mf.log_param('number of batches', number_of_batches)