ml_final_project.py

# -*- coding: utf-8 -*-
"""ML Final Project

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1Aof3bcIIqSmvsh0cux6wZ5NPk1wY-l3D

### install dependencies
"""

!gdown "1W3-WEplVSztLR3lvkyYdiKZGMT4y0cNi&confirm=t"

#!unzip IMDB.zip

#!pip install mlflow

"""# Content-based filtering

### import libraries
"""

import numpy as np
import pandas as pd
import mlflow as mf

#mf.log_artifacts({'rating':'/content/rating_small.csv', 'rating':'/content/rating_small.csv', 'movies':'/content/movies_metadata.csv','keywords':'/content/keywords.csv', 'credits':'/content/credits.csv'})

"""### read data from file"""

keywords = pd.read_csv('/content/IMDB/keywords.csv')
keywords

rating = pd.read_csv('/content/IMDB/ratings_small.csv')
rating

credits = pd.read_csv('/content/IMDB/credits.csv')
credits

metadata = pd.read_csv('/content/IMDB/movies_metadata.csv')
metadata

"""keep only related columns from released movies:"""

metadata = metadata[metadata['status'] == 'Released']
cols = np.array(['adult', 'belongs_to_collection', 'genres', 'id', 'original_language', 'title', 'production_countries', 'production_companies', 'video']) 
metadata = metadata[cols]

metadata.iloc[1]

def find_collection(x):
    if x == '':
        return ''
    return eval(str(x))['name']

metadata['belongs_to_collection'] = metadata['belongs_to_collection'].fillna('')
metadata['belongs_to_collection'] = metadata['belongs_to_collection'].apply(find_collection)
metadata.iloc[1]

def find_names(x):
    if x == '':
        return ''
    genre_arr = eval(str(x))
    return ','.join(i['name'] for i in eval(str(x)))
    
metadata['genres'] = metadata['genres'].fillna('')
metadata['genres']=metadata['genres'].apply(find_names)
metadata['production_countries']=metadata['production_countries'].apply(find_names)
metadata['production_companies']=metadata['production_companies'].apply(find_names)
credits['cast'] = credits['cast'].apply(find_names)
metadata.iloc[1]

keywords['keywords'] = keywords['keywords'].apply(find_names)
metadata['id'] = metadata['id'].astype(int)
metadata = pd.merge(metadata,keywords,how='inner',on='id')
metadata.iloc[1]

def to_int(x):
    if x == 'True':
        return 1
    return 0

metadata['adult'].unique()

"""there are 3 values other than True or False in adult column. there are entered by mistake so we remove those rows."""

metadata = metadata[(metadata['adult'] == 'True') | (metadata['adult'] == 'False')]
metadata['adult'] = metadata['adult'].apply(to_int)
metadata['video'].unique()

"""removing nan values from dataset and replacing 'True' and 'False' with 1 and 0:"""

metadata = metadata[~metadata['video'].isna()]
metadata['video'] = metadata['video'].apply(to_int)

"""## Vectorize string features"""

metadata

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer

def my_tok(text):
    return text.split(",")

def vectorize_string(col_name, feature_name, limit=None, df=metadata):
    vectorizer = CountVectorizer(tokenizer=my_tok, max_features=limit, min_df=2)
    X = vectorizer.fit_transform(df[col_name])
    vec_cols = vectorizer.get_feature_names_out()
    vec_data = X.toarray()
    #vec_cols = np.char.add(feature_name+':', vec_cols)
    vec_cols = feature_name+':'+vec_cols
    return vec_data, vec_cols

def tfidf(col_name, feature_name, limit=None, df=metadata):
    vectorizer = TfidfVectorizer(tokenizer=my_tok, max_features=limit, min_df=2)
    X = vectorizer.fit_transform(df[col_name])
    vec_cols = vectorizer.get_feature_names_out()
    vec_data = X.toarray()
    #vec_cols = np.char.add(feature_name+':', vec_cols)
    vec_cols = feature_name+':'+vec_cols
    return vec_data, vec_cols

genre_data, genre_cols = vectorize_string('genres', 'genre')
genre_cols

companies_data, companies_cols = vectorize_string('production_companies', 'company', 100)
companies_cols

countries_data, countries_cols = vectorize_string('production_countries', 'country')
countries_cols

collection_data, collection_cols = vectorize_string('belongs_to_collection', 'collection')
collection_cols

metadata['original_language']= metadata['original_language'].fillna('')
lang_data, lang_cols = vectorize_string('original_language', 'lang')
lang_cols

collection_cols.shape

keyword_data, keyword_cols = tfidf('keywords', 'keyword', 1000)
keyword_cols

credits.drop(columns=['crew'], inplace=True)
credit_data, credit_cols = vectorize_string('cast','cast', 1000, df=credits)
credit_cols

metadata = pd.concat([metadata[['title','id','adult','video']], 
                      pd.DataFrame(genre_data, columns=genre_cols),
                      pd.DataFrame(countries_data, columns=countries_cols),
                      pd.DataFrame(collection_data, columns=collection_cols),
                      pd.DataFrame(keyword_data, columns=keyword_cols),
                      pd.DataFrame(companies_data, columns=companies_cols),
                      pd.DataFrame(lang_data, columns=lang_cols)], axis=1)

credits[credit_cols] = credit_data
metadata = pd.merge(metadata, credits, how='inner', on='id')
metadata

#metadata.drop(['production_countries', 'genres', 'belongs_to_collection', 'keywords', 'production_companies', 'original_language'], axis=1, inplace=True)

"""list of all numerical features(everything except id and title)"""

feature_cols = np.concatenate((np.array(['adult', 'video']), genre_cols,countries_cols,collection_cols,keyword_cols,companies_cols,lang_cols,credit_cols))
feature_cols
#metadata[feature_cols] = metadata[feature_cols].astype('int8')

del genre_data,countries_data,collection_data,keyword_data,companies_data,lang_data,credit_data
del genre_cols,countries_cols,collection_cols,keyword_cols,companies_cols,lang_cols,credit_cols

feature_cols.shape

metadata

def split_dataframe(df, holdout_fraction=0.1):
  test = df.sample(frac=holdout_fraction, replace=False)
  train = df[~df.index.isin(test.index)]
  return train, test

train, test = split_dataframe(metadata)

allIds = metadata['id']

number_of_batches = 4
batches = np.array_split(train, number_of_batches)
mf.log_param('number of batches', number_of_batches)
del metadata
del train

"""## Algorithm

"""

batches[0]

from sklearn.metrics.pairwise import cosine_similarity

"""`content_based_recommmeder` returns a list of movie ids based on it's input. the input should be a dataframe which has `movieId`, `rating` columns(like `ratings_small.csv` but without `userId`)"""

number_of_batches =1
def content_based_recommender_movie(movieId):
    print("movie title is:", metadata[metadata['id']==movieId])
    sim_mat= cosine_similarity(metadata[feature_cols])
    return sim_mat

#content_based_recommender_movie(272)

batches[1].describe()

from sklearn.metrics.pairwise import euclidean_distances as dist
def content_based_recommender(user, df, k=10, movieIds=allIds):
    user_movies = pd.merge(user,df,how='inner',left_on='movieId',right_on='id')
    user_movies[feature_cols] = user_movies[feature_cols].multiply(user_movies['rating'], axis="index")
    mean_user_movies = user_movies[feature_cols].mean(axis=0)
    sim_mat = cosine_similarity(df[feature_cols][df.id.isin(movieIds)], mean_user_movies[feature_cols].values.reshape(1,-1))
    temp_data = {'id':df['id'][df.id.isin(movieIds)], 'title':df['title'][df.id.isin(movieIds)], 'sim':sim_mat.flatten()}
    return pd.DataFrame(temp_data)

def content_based_all_batches(user, k=10, movieIds=allIds):
    ans = content_based_recommender(user, batches[0], k, movieIds)
    for i in range(1,number_of_batches):
        ans.append(content_based_recommender(user, batches[i], k, movieIds))
    return ans.sort_values(by='sim', ascending=False)
    

content_based_k = 10
mf.log_param('content based k', content_based_k)
#xx = content_based_recommender(rating[rating['userId'] == 1], batches[1], content_based_k)
xx = content_based_all_batches(rating[rating['userId'] == 1], content_based_k)
xx.shape

"""# Collaborative Filtering

### import libraries
"""

import numpy as np
import pandas as pd
from sklearn.utils.extmath import randomized_svd

"""### explore datasets"""

rating = pd.read_csv('/content/IMDB/ratings_small.csv')
rating.head()

rating.shape

links_small = pd.read_csv('/content/IMDB/links_small.csv')
links_small.head()

credits = pd.read_csv('/content/IMDB/credits.csv')
credits.head()

movie = pd.read_csv('/content/IMDB/movies_metadata.csv')
movie.head()

movie = movie.rename(columns={'id': 'movieId'})

movie.shape

movie.head()

"""### data preprocessing

There are three rows entered by mistake, so we remove that row.
"""

movie = movie[(movie['movieId']!='1997-08-20') & (movie['movieId']!='2012-09-29') & (movie['movieId']!='2014-01-01')]

def find_names(x):
    if x == '':
        return ''
    genre_arr = eval(str(x))
    return ','.join(i['name'] for i in eval(str(x)))
    
movie['genres'] = movie['genres'].fillna('')

movie['genres']=movie['genres'].apply(find_names)

movie.movieId = movie.movieId.astype("uint64")

"""only keep rating for movies with metadata in movie dataset"""

new_rating = pd.merge(rating, movie,  how='inner', on=["movieId"])

new_rating = new_rating[["userId", "movieId", "rating"]]

movie.head()

new_rating.head()

train, test = split_dataframe(new_rating)

"""### matrix factorization"""

inter_mat_df = rating.pivot(index = 'userId', columns ='movieId', values = 'rating').fillna(0)
inter_mat_df

inter_mat = inter_mat_df.to_numpy()

ratings_mean = np.mean(inter_mat, axis = 1)
inter_mat_normal = inter_mat - ratings_mean.reshape(-1, 1)

inter_mat_normal

"""We use singular value decomposition for matrix factorization"""

svd_U, svd_sigma, svd_V = randomized_svd(inter_mat_normal, 
                              n_components=15,
                              n_iter=5,
                              random_state=47)

"""This function gives the diagonal form"""

svd_sigma = np.diag(svd_sigma)

"""Making predictions"""

rating_weights = np.dot(np.dot(svd_U, svd_sigma), svd_V) + ratings_mean.reshape(-1, 1)

weights_df = pd.DataFrame(rating_weights, columns = inter_mat_df.columns)

weights_df.head()

"""making recommendations"""

def recommend_top_k(preds_df, ratings_df, movie, userId, k=10):
    user_row = userId-1 
    sorted_user_predictions = preds_df.iloc[user_row].sort_values(ascending=False) 
    user_data = ratings_df[ratings_df.userId == (userId)]
    user_rated = user_data.merge(movie, how = 'left', left_on = 'movieId', right_on = 'movieId'). \
                  sort_values(['rating'], ascending=False)
    user_preds = movie.merge(pd.DataFrame(sorted_user_predictions).reset_index(), how = 'left',
               on = 'movieId').rename(columns = {user_row: 'prediction'}). \
               sort_values('prediction', ascending = False). \
               iloc[:k, :]
    return user_rated, user_preds

collaborative_k = 100
user_rated, user_preds = recommend_top_k(weights_df, new_rating, movie, 220, collaborative_k)
mf.log_param('collaborative k', collaborative_k)

user_preds.head()

user_rated.head()

user_rated[["title", "genres"]].head(10)

user_preds[["title", "genres"]].head(10)

"""# Ensemble Model"""

def ensemble(userId, k=10):
    user_rated, user_preds = recommend_top_k(weights_df, new_rating, movie, userId, k*k)
    content_based_result = content_based_all_batches(rating[rating['userId'] == userId], k=k, movieIds=user_preds['movieId'])
    return content_based_result[['id','title']]

ensemble_k=10
mf.log_param('ensemble k', ensemble_k)
ensemble(220, ensemble_k)

"""# Evaluation"""

df_res = user_preds[["movieId", "prediction"]]. \
          merge(user_rated[["movieId", "rating"]], how = 'outer', on = 'movieId')

df_res.sort_values(by='prediction',ascending=False,inplace=True)
df_res

threshold = 2
df_res['prediction'] = df_res['prediction'] >= threshold
df_res['rating'] = df_res['rating'] >= threshold
df_res

def precision_at_k(df, k=10, y_test: str='rating', y_pred='prediction'):  
    dfK = df.head(k)
    sum_df = dfK[y_pred].sum()
    true_pred = dfK[dfK[y_pred] & dfK[y_test]].shape[0]
    if sum_df > 0:
        return true_pred/sum_df
    else:
        return None

def recall_at_k(df, k=10, y_test='rating', y_pred='prediction'):
    dfK = df.head(k)
    sum_df = df[y_test].sum()
    true_pred = dfK[dfK[y_pred] & dfK[y_test]].shape[0]
    if sum_df > 0:
        return true_pred/sum_df
    else:
        return None

prec_at_k = precision_at_k(df_res, 100, y_test='rating', y_pred='prediction')
rec_at_k = recall_at_k(df_res, 100, y_test='rating', y_pred='prediction')

print("precision@k: ", prec_at_k)
print("recall@k: ", rec_at_k)
mf.log_metric('recall', rec_at_k)
mf.log_metric('precision', prec_at_k)



"""# MLOps"""

def updata_batch(new_batch):
    number_of_batches = number_of_batches+1
    batches = batches.append(new_batch)
    mf.log_param('number of batches', number_of_batches)