import mlflow from mlflow import log_metric, log_param, log_artifacts import numpy as np import pandas as pd import matplotlib.pyplot as plt from ast import literal_eval from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer from sklearn.metrics.pairwise import linear_kernel, cosine_similarity from nltk.stem.snowball import SnowballStemmer import warnings; warnings.simplefilter('ignore') """# Read datasets""" # /Applications/Education/7th Semester/Machine Learning/Me/Project/IMDB/credits.csv def read_data_set(): md=pd.read_csv('IMDB/movies_metadata.csv') md.head(2) credits=pd.read_csv('IMDB/credits.csv') credits.head(2) keywords=pd.read_csv('IMDB/keywords.csv') keywords.head(2) links_small=pd.read_csv('IMDB/links_small.csv') links_small.head(2) df_rating = pd.read_csv('IMDB/ratings_small.csv') return md,credits,keywords,links_small,df_rating df_rating.head(5) """# Data preprocessing""" def data_preprocessing(): links_small = links_small[links_small['tmdbId'].notnull()]['tmdbId'].astype('int') md['genres'] = md['genres'].fillna('[]').apply(literal_eval).apply(lambda x: [i['name'] for i in x] if isinstance(x, list) else []) vote_counts = md[md['vote_count'].notnull()]['vote_count'].astype('int') vote_averages = md[md['vote_average'].notnull()]['vote_average'].astype('int') C = vote_averages.mean() m = vote_counts.quantile(0.95) md['year'] = pd.to_datetime(md['release_date'], errors='coerce').apply(lambda x: str(x).split('-')[0] if x != np.nan else np.nan) qualified = md[(md['vote_count'] >= m) & (md['vote_count'].notnull()) & (md['vote_average'].notnull())][['title', 'year', 'vote_count', 'vote_average', 'popularity', 'genres']] qualified['vote_count'] = qualified['vote_count'].astype('int') qualified['vote_average'] = qualified['vote_average'].astype('int') qualified.shape def weighted_rating(x): v = x['vote_count'] R = x['vote_average'] return (v/(v+m) * R) + (m/(m+v) * C) s = md.apply(lambda x: pd.Series(x['genres']),axis=1).stack().reset_index(level=1, drop=True) s.name = 'genre' gen_md = md.drop('genres', axis=1).join(s) md = md.drop([19730, 29503, 35587]) md.head(5) md['id'] = md['id'].astype('int') smd = md[md['id'].isin(links_small)] smd.shape smd['tagline'] = smd['tagline'].fillna('') smd['description'] = smd['overview'] + smd['tagline'] smd['description'] = smd['description'].fillna('') tf = TfidfVectorizer(analyzer='word',ngram_range=(1, 2),min_df=0, stop_words='english') tfidf_matrix = tf.fit_transform(smd['description']) tfidf_matrix.shape cosine_sim = linear_kernel(tfidf_matrix, tfidf_matrix) smd = smd.reset_index() titles = smd['title'] indices = pd.Series(smd.index, index=smd['title']) """#First content based recommendation""" def get_recommendations(title): idx = indices[title] sim_scores = list(enumerate(cosine_sim[idx])) sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True) sim_scores = sim_scores[1:31] movie_indices = [i[0] for i in sim_scores] return titles.iloc[movie_indices] # get_recommendations('The Godfather').head(10) """# Some process on data to have better recommendations""" read_data_set.keywords['id'] = read_data_set.keywords['id'].astype('int') credits['id'] = credits['id'].astype('int') read_data_set.md['id'] = read_data_set.md['id'].astype('int') read_data_set.md.shape md = md.merge(credits, on='id') md = md.merge(keywords, on='id') smd = md[md['id'].isin(read_data_set.links_small)] smd.shape smd['cast'] = smd['cast'].apply(literal_eval) smd['crew'] = smd['crew'].apply(literal_eval) smd['keywords'] = smd['keywords'].apply(literal_eval) smd['cast_size'] = smd['cast'].apply(lambda x: len(x)) smd['crew_size'] = smd['crew'].apply(lambda x: len(x)) def get_director(x): for i in x: if i['job'] == 'Director': return i['name'] return np.nan smd['director'] = smd['crew'].apply(get_director) smd['cast'] = smd['cast'].apply(lambda x: [i['name'] for i in x] if isinstance(x, list) else []) smd['cast'] = smd['cast'].apply(lambda x: x[:3] if len(x) >=3 else x) smd['keywords'] = smd['keywords'].apply(lambda x: [i['name'] for i in x] if isinstance(x, list) else []) smd['cast'] = smd['cast'].apply(lambda x: [str.lower(i.replace(" ", "")) for i in x]) smd['director'] = smd['director'].astype('str').apply(lambda x: str.lower(x.replace(" ", ""))) smd['director'] = smd['director'].apply(lambda x: [x,x, x]) s = smd.apply(lambda x: pd.Series(x['keywords']),axis=1).stack().reset_index(level=1, drop=True) s.name = 'keyword' s = s.value_counts() s[:5] s = s[s > 1] stemmer = SnowballStemmer('english') stemmer.stem('dogs') def filter_keywords(x): words = [] for i in x: if i in s: words.append(i) return words smd['keywords'] = smd['keywords'].apply(filter_keywords) smd['keywords'] = smd['keywords'].apply(lambda x: [stemmer.stem(i) for i in x]) smd['keywords'] = smd['keywords'].apply(lambda x: [str.lower(i.replace(" ", "")) for i in x]) smd['soup'] = smd['keywords'] + smd['cast'] + smd['director'] + smd['genres'] smd['soup'] = smd['soup'].apply(lambda x: ' '.join(x)) smd.head(5) count = CountVectorizer(analyzer='word',ngram_range=(1, 2),min_df=0, stop_words='english') count_matrix = count.fit_transform(smd['soup']) cosine_sim = cosine_similarity(count_matrix, count_matrix) smd = smd.reset_index() titles = smd['title'] indices = pd.Series(smd.index, index=smd['title']) df_rating = pd.read_csv('IMDB/ratings_small.csv') df_rating.head(5) """# Data preprocessing and analyzing""" rating_copy = df_rating.copy() rating_copy['rating'] = rating_copy['rating'].apply(np.floor) gp_by_rating = rating_copy.groupby('rating')['rating'].agg(['count']) movie_count = df_rating['movieId'].nunique() cust_count = df_rating['userId'].nunique() ax = gp_by_rating.plot(kind = 'barh', legend = False, figsize = (8,8)) plt.title('{:,} Movies, {:,} customers'.format(movie_count, cust_count), fontsize=14) plt.axis('off') for i in range(0,6): ax.text(gp_by_rating.iloc[i][0]/4, i, 'Rating {}: {:.0f}%'. format(i, gp_by_rating.iloc[i][0]*100 / gp_by_rating.sum()[0]), color = 'black') agg_function = ['count','mean'] gp_by_movie = df_rating.groupby('movieId')['rating'].agg(agg_function) df_rating = pd.merge(df_rating, smd, how='right', left_on='movieId', right_on='id') df_rating = df_rating[['movieId', 'userId', 'rating']] pivot_rating = pd.pivot_table(df_rating, values='rating', index='userId', columns='movieId') pivot_rating """# Improved recommendation""" with mlflow.start_run(run_name="run") as run: get_recommendations('The Dark Knight').head(10) def improved_recommendations(title): idx = indices[title] sim_scores = list(enumerate(cosine_sim[idx])) sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True) sim_scores = sim_scores[1:26] movie_indices = [i[0] for i in sim_scores] eval_cosine = sum(movie_indices) / len(movie_indices) movies = smd.iloc[movie_indices][['title', 'vote_count', 'vote_average', 'year']] vote_counts = movies[movies['vote_count'].notnull()]['vote_count'].astype('int') vote_averages = movies[movies['vote_average'].notnull()]['vote_average'].astype('int') C = vote_averages.mean() m = vote_counts.quantile(0.60) qualified = movies[(movies['vote_count'] >= m) & (movies['vote_count'].notnull()) & (movies['vote_average'].notnull())] qualified['vote_count'] = qualified['vote_count'].astype('int') qualified['vote_average'] = qualified['vote_average'].astype('int') qualified['wr'] = qualified.apply(weighted_rating, axis=1) qualified = qualified.sort_values('wr', ascending=False).head(10) return qualified, eval_cosine q, eval_cosine = improved_recommendations('The Dark Knight') mlflow.log_metric('cosine_sim',eval_cosine) """# Collaborative Filtering""" """# PearsonR recommendation""" df_movie_title = smd[['id', 'title']] df_movie_title.shape def corr_recommend(movie_title, min_count): i = int(df_movie_title[df_movie_title['title'] == movie_title]['id']) target = pivot_rating[i] similar_to_target = pivot_rating.corrwith(target) corr_target = pd.DataFrame(similar_to_target, columns = ['PearsonR']) corr_target.dropna(inplace = True) corr_target = corr_target.sort_values('PearsonR', ascending = False) corr_target.index = corr_target.index.map(int) corr_target = corr_target.join(df_movie_title).join(gp_by_movie)[['PearsonR', 'title', 'count', 'mean']] return corr_target[corr_target['count']>min_count][:10] corr_recommend('The Dark Knight', 0) def hybrid_recommendation(movie_name): soup_based = improved_recommendations(movie_name) corr = corr_recommend(soup_based.iloc[0]['title'],0) return get_recommendations(corr.iloc[0]['title']) print(hybrid_recommendation('Toy Story').head(10))