Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<count_values>
%matplotlib inline
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 warnings.filterwarnings("ignore" )
train_df['rez_esc'].dropna().value_counts()<feature_engineering>
data = pd.read_csv(".. /input/tmdb-box-office-prediction/train.csv")
 print(data.shape)
 data.head(n=2 )
train_df.loc[pd.isnull(train_df['rez_esc']), 'rez_esc'] = 0<filter>
data_explore = data.copy()
train_df.loc[pd.isnull(train_df['meaneduc']),('edjefa', 'edjefe', 'escolari', 'meaneduc')]<feature_engineering>
data_explore.isna().sum()
train_df.loc[pd.isnull(train_df['meaneduc']), 'meaneduc'] = train_df.loc[pd.isnull(train_df['meaneduc']), 'escolari']<feature_engineering>
data_explore['is_sequel'] = data_explore['belongs_to_collection'].apply(lambda x: 0 if pd.isna(x)else 1 ).astype('int64' )
train_df.loc[pd.isnull(train_df['SQBmeaned']), 'SQBmeaned'] = train_df.loc[pd.isnull(train_df['SQBmeaned']), 'meaneduc']**2<filter>
def modify_date(x):

 x=str(x)
 year=x.split('/')[2]
if int(year)<20:
return x[:-2]+'20'+year
else:
return x[:-2]+'19'+year
data_explore['release_date']=data_explore['release_date'].apply(lambda x: modify_date(x))
data_explore['release_year'] = pd.DatetimeIndex(data_explore['release_date'] ).year
data_explore['release_month'] = pd.DatetimeIndex(data_explore['release_date'] ).month
data_explore['release_day'] = pd.DatetimeIndex(data_explore['release_date'] ).day
data_explore['release_dow'] = pd.DatetimeIndex(data_explore['release_date'] ).dayofweek
len(train_df.loc[train_df['age'] == 0].index )<feature_engineering>
drop_cols = ['id', 'belongs_to_collection', 'homepage', 'imdb_id', 'release_date', 'poster_path', 'tagline', 'title']
data_explore = data_explore.drop(columns=drop_cols, axis=1 )
train_df = train_df.loc[train_df['age']!=0]<groupby>
nan_cols = data_explore.isna().sum()
nan_cols[nan_cols>0]
train_df.groupby('dependency' ).size()<data_type_conversions>
dict_cols = ['genres', 'production_companies', 'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
for col in dict_cols:
data_explore[col] = data_explore[col].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
mode = train_df.loc[(train_df['dependency'] != 'yes')&(train_df['dependency'] != 'no'), 'dependency'].astype(float ).mode()
mode<data_type_conversions>
na_cols = data_explore.columns[data_explore.isna().any() ].tolist()
na_cols.remove('overview')
 na_cols.remove('runtime')
 data_explore['runtime'].fillna(value=data_explore['runtime'].median() , inplace=True)
 data_explore['overview'].fillna(value='', inplace=True)
 for col in na_cols:
data_explore[col].fillna(value='', inplace=True )
def mutate_columns(df):
df.loc[df['dependency']=='no', 'dependency'] = np.sqrt(df['SQBdependency'])
 df.loc[df['dependency']=='yes', 'dependency'] = np.sqrt(df['SQBdependency'])
 df['dependency'] = df['dependency'].astype('float16')
 df.loc[df['edjefe']=='no', 'edjefe'] = 0
df.loc[df['edjefe']=='yes', 'edjefe'] = 4
df['edjefe'] = df['edjefe'].astype('uint8')
 df.loc[df['edjefa']=='no', 'edjefa'] = 0
df.loc[df['edjefa']=='yes', 'edjefa'] = 4
df['edjefa'] = df['edjefa'].astype('uint8' )<categorify>
def get_names(x, col):

 names = []
for item in x:
if col=='crew':
if item['job']=='Director':
names.append(item['name'])
 elif col=='cast':
if item['order'] in(0, 1, 2):
names.append(item['name'])
 else:
names.append(item['name'])
 return names
for col in dict_cols:
data_explore[col] = data_explore[col].apply(lambda x: get_names(x, col))
tdf = train_df[['Target']]
n_train_df = train_df
for col in ['v18q', 'refrig', 'computer', 'television', 'mobilephone', 'v14a']:
n_train_df[col] = n_train_df[col].astype('category')
 dfcat = pd.get_dummies(n_train_df[[ 'v18q', 'refrig', 'computer', 'television', 'mobilephone', 'v14a']])
 df_ = pd.concat([dfcat, tdf], axis=1 )<groupby>
Q1 = data_explore.quantile(0.25)
 Q3 = data_explore.quantile(0.75)
 IQR = Q3 - Q1
outliers =(( data_explore <(Q1 - 1.5 * IQR)) |(data_explore >(Q3 + 1.5 * IQR)) ).sum()
outliers[outliers>0]
df_ = df_.groupby(['Target'] ).sum()
df_.reset_index(inplace = True )<feature_engineering>
most_popular_movies = data_explore.sort_values('popularity', ascending=False ).head(n=20)
 most_popular_movies['revenue(million)'] = most_popular_movies['revenue'].apply(lambda x : x//1000000)
 most_popular_movies['budget(million)'] = most_popular_movies['budget'].apply(lambda x : x//1000000)
 most_popular_movies[['genres', 'original_title', 'production_companies', 'popularity', 'cast', 'crew', 'budget(million)', 'revenue(million)']]
df_q = train_df[['Target', 'epared1', 'epared2', 'epared3', 'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3']]
df_q.loc[df_q['epared1'] == 1, 'wall'] = 'Bad'
df_q.loc[df_q['epared2'] == 1, 'wall'] = 'Regular'
df_q.loc[df_q['epared3'] == 1, 'wall'] = 'Good'
df_q.loc[df_q['etecho1'] == 1, 'roof'] = 'Bad'
df_q.loc[df_q['etecho2'] == 1, 'roof'] = 'Regular'
df_q.loc[df_q['etecho3'] == 1, 'roof'] = 'Good'
df_q.loc[df_q['eviv1'] == 1, 'floor'] = 'Bad'
df_q.loc[df_q['eviv2'] == 1, 'floor'] = 'Regular'
df_q.loc[df_q['eviv3'] == 1, 'floor'] = 'Good'
df_q = df_q[['Target', 'wall', 'roof', 'floor']]<data_type_conversions>
highest_revenue_movies = data_explore.sort_values('revenue', ascending=False ).head(n=20)
 highest_revenue_movies['revenue(million)'] = highest_revenue_movies['revenue'].apply(lambda x : x//1000000)
 highest_revenue_movies['budget(million)'] = highest_revenue_movies['budget'].apply(lambda x : x//1000000)
 highest_revenue_movies[['genres', 'original_title', 'production_companies', 'popularity', 'cast', 'crew', 'budget(million)', 'revenue(million)']]
def feature_engineer(x):
x['escolari_age'] = x['escolari'] / x['age']
x['refrig'] = x['refrig'].astype(int)
 x['computer'] = x['computer'].astype(int)
 x['television'] = x['television'].astype(int)
 x['mobilephone'] = x['mobilephone'].astype(int)
 x['v14a'] = x['v14a'].astype(int)
 x['v18q'] = x['v18q'].astype(int)
 x['epared1'] = x['epared1'].astype(int)
 x['epared2'] = x['epared2'].astype(int)
 x['epared3'] = x['epared3'].astype(int)
 x['etecho1'] = x['etecho1'].astype(int)
 x['etecho2'] = x['etecho2'].astype(int)
 x['etecho3'] = x['etecho3'].astype(int)
 x['eviv1'] = x['eviv1'].astype(int)
 x['eviv2'] = x['eviv2'].astype(int)
 x['eviv3'] = x['eviv3'].astype(int)
 x['abastaguadentro'] = x['abastaguadentro'].astype(int)
 x['abastaguafuera'] = x['abastaguafuera'].astype(int)
 x['abastaguano'] = x['abastaguano'].astype(int)
 x['abastaguano'] = x['abastaguano'].astype(int)
 x[['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']] = x[['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']].apply(pd.to_numeric)
 x['appliances'] =(x['refrig'] + x['computer'] + x['television'])
 x['rent_by_hhsize'] = x['v2a1'] / x['hhsize']
x['rent_by_people'] = x['v2a1'] / x['r4t3']
x['rent_by_rooms'] = x['v2a1'] / x['rooms']
x['rent_by_living'] = x['v2a1'] / x['tamviv']
x['rent_by_minor'] = x['v2a1'] / x['hogar_nin']
x['rent_by_adult'] = x['v2a1'] / x['hogar_adul']
x['rent_by_dep'] = x['v2a1'] / x['dependency']
x['rent_by_head_educ'] = x['v2a1'] /(x['edjefe'] + x['edjefa'])
 x['rent_by_educ'] = x['v2a1'] / x['meaneduc']
x['rent_by_numPhone'] = x['v2a1'] / x['qmobilephone']
x['rent_by_gadgets'] = x['v2a1'] /(x['computer'] + x['mobilephone'] + x['v18q'])
 x['rent_by_num_gadgets'] = x['v2a1'] /(x['v18q1'] + x['qmobilephone'])
 x['rent_by_appliances'] = x['v2a1'] / x['appliances']
x['under12'] = x['r4t1']/x['r4t3']
x['under12_male'] = x['r4h1']/x['r4t3']
x['under12_female'] = x['r4m1']/x['r4t3']
x['Proportion_male'] = x['r4h3']/x['r4t3']
x['Proportion_female'] = x['r4m3']/x['r4t3']
x['tablet_density'] = x['v18q1'] / x['r4t3']
x['phone_density'] = x['qmobilephone'] / x['r4t3']
x['wall_qual'] = x['epared3'] - x['epared1']
x['roof_qual'] = x['etecho3'] - x['etecho1']
x['floor_qual'] = x['eviv3'] - x['eviv1']
x['water_qual'] = x['abastaguadentro'] - x['abastaguano']
x['house_qual'] = x['wall_qual'] + x['roof_qual'] + x['floor_qual']
x['person_per_room'] = x['hhsize'] / x['rooms']
x['person_per_appliances'] = x['hhsize'] / x['appliances']
x['educ_qual'] =(1 * x['instlevel1'])+(2 * x['instlevel2'])+(3 * x['instlevel3'])+(4 * x['instlevel4'])+(5 * x['instlevel5'])+(6 * x['instlevel6'])+(7 * x['instlevel7'])+(8 * x['instlevel8'])+(9 * x['instlevel9'])
 x['educ_by_individual'] = x['escolari']/x['r4t3']
x['educ_by_adult'] = x['escolari']/(x['r4t3'] - x['r4t1'])
 x['educ_by_child'] = x['escolari']/x['r4t1']
x['max_educ'] = np.max(x[['edjefa','edjefe']])
 def reverse_label_encoding(row, df):
for c in df.columns:
if row[c] == 1:
return int(c[-1])
 def rate_sanitary(row, df):
c = df.columns.tolist() [0]
if row[c] == 'sanitario2':
return 3
elif row[c] == 'sanitario3':
return 2
elif row[c] == 'sanitario5':
return 1
else:
return 0
def rate_cooking(row, df):
c = df.columns.tolist() [0]
if row[c] == 'energcocinar2':
return 3
elif row[c] == 'energcocinar3':
return 2
elif row[c] == 'energcocinar4':
return 1
else:
return 0
def rate_rubbish(row, df):
c = df.columns.tolist() [0]
if row[c] == 'elimbasu1':
return 1
elif row[c] == 'elimbasu2':
return 2
else:
return 0
x['sanitary'] = x.apply(lambda q: reverse_label_encoding(q, x[['sanitario1', 'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6']]), axis=1)
 x['cooking'] = x.apply(lambda q: reverse_label_encoding(q, x[['energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4']]), axis=1)
 x['rubbish'] = x.apply(lambda q: reverse_label_encoding(q, x[['elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4', 'elimbasu5', 'elimbasu6']]), axis=1)
 x['region'] = x.apply(lambda q: reverse_label_encoding(q, x[['lugar1', 'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6']]), axis=1)
 x['sanitary_i'] = x.apply(lambda q: rate_sanitary(q, x[['sanitary']]), axis = 1)
 x['cooking_i'] = x.apply(lambda q: rate_cooking(q, x[['cooking']]), axis = 1)
 x['rubbish_i'] = x.apply(lambda q: rate_rubbish(q, x[['rubbish']]), axis = 1)
 x['zone'] = x['area1'] - x['area2']
x.replace([np.inf, -np.inf], 0, inplace = True)
 x.fillna(0, inplace = True )<feature_engineering>
highest_budget_movies = data_explore.sort_values('budget', ascending=False ).head(n=20)
 highest_budget_movies['revenue(million)'] = highest_budget_movies['revenue'].apply(lambda x : x//1000000)
 highest_budget_movies['budget(million)'] = highest_budget_movies['budget'].apply(lambda x : x//1000000)
 highest_budget_movies[['genres', 'original_title', 'production_companies', 'popularity', 'cast', 'crew', 'budget(million)', 'revenue(million)']]
feature_engineer(train_df)
 feature_engineer(test_df )<define_variables>
most_profit_movies = data_explore.copy()
most_profit_movies['revenue(million)'] = most_profit_movies['revenue'].apply(lambda x : x//1000000)
 most_profit_movies['budget(million)'] = most_profit_movies['budget'].apply(lambda x : x//1000000)
 most_profit_movies['profit(million)'] = most_profit_movies['revenue(million)']-most_profit_movies['budget(million)']
most_profit_movies = most_profit_movies.sort_values('profit(million)', ascending=False ).head(n=20)
 most_profit_movies[['genres', 'original_title', 'production_companies', 'popularity', 'cast', 'crew', 'budget(million)', 'revenue(million)', 'profit(million)']]
def agg_features(y):
mean_list = []
count_list = ['escolari_age', 'phone_density', 'rez_esc', 'dis', 'male', 'female','v2a1','house_qual', 'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total', 'escolari','estadocivil1','estadocivil2','estadocivil3','estadocivil4','estadocivil5','estadocivil6','estadocivil7','parentesco1', 'parentesco2', 'parentesco3',
'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10', 'parentesco11', 'parentesco12']
for group in ['idhogar']:
for item in count_list:
for agg_m in ['mean','std','min','max','sum']:
id_agg = y[item].groupby(y[group] ).agg(agg_m)
 new_col = item + '_' + agg_m + '_' + group
y[new_col] = id_agg
drop_ = ['sanitary', 'cooking', 'rubbish', 'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9',
'sanitario1', 'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6', 'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4', 'elimbasu5', 'elimbasu6', 'tamhog', 'tamviv', 'hhsize', 'v18q',
'v14a', 'agesq','mobilephone', 'female', 'estadocivil1','estadocivil2','estadocivil3','estadocivil4','estadocivil5','estadocivil6','estadocivil7', 'parentesco2', 'parentesco3',
'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10', 'parentesco11', 'parentesco12', 'lugar1', 'lugar2', 'lugar3', 'lugar4',
'lugar5', 'lugar6', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned',
'r4t1', 'r4t2', 'r4t3']
y.drop(( drop_), inplace = True, axis = 1)
 y.replace([np.inf, -np.inf], 0, inplace = True)
 y.fillna(0, inplace = True)
 return y 
<feature_engineering>
genres_info = []
for col in genres:
total_movies, total_budget, median_budget, total_revenue, median_revenue, median_popularity=0, 0, 0, 0, 0, 0
total_movies = data_explore_genres[data_explore_genres[col]==1][col].count()
total_budget = data_explore_genres[data_explore_genres[col]==1]['budget'].sum()
median_budget = data_explore_genres[data_explore_genres[col]==1]['budget'].median()
total_revenue = data_explore_genres[data_explore_genres[col]==1]['revenue'].sum()
median_revenue = data_explore_genres[data_explore_genres[col]==1]['revenue'].median()
median_popularity = data_explore_genres[data_explore_genres[col]==1]['popularity'].median()
genres_info.append([col, total_movies, total_budget, median_budget, total_revenue, median_revenue, median_popularity] )
train_df = agg_features(train_df)
 test_df = agg_features(test_df )<data_type_conversions>
genres_info = pd.DataFrame(genres_info, columns=['genres', 'movies_count', 'total_budget', 'median_budget', 'total_revenue', 'median_revenue', 'median_popularity'])
 genres_info['total_budget(million)'] = genres_info['total_budget'].apply(lambda x : x//1000000)
 genres_info['median_budget(million)'] = genres_info['median_budget'].apply(lambda x : x//1000000)
 genres_info['total_revenue(million)'] = genres_info['total_revenue'].apply(lambda x : x//1000000)
 genres_info['median_revenue(million)'] = genres_info['median_revenue'].apply(lambda x : x//1000000)
 genres_info[['genres', 'movies_count', 'total_budget(million)', 'median_budget(million)', 'total_revenue(million)', 'median_revenue(million)', 'median_popularity']]
train_df = train_df.loc[train_df['parentesco1'] == 1]
train_df.fillna(value=0, inplace=True )<prepare_x_and_y>
from wordcloud import WordCloud
y = train_df[['Target']]
x = train_df.drop(['Target','Id','idhogar'], axis = 1 )<data_type_conversions>
corr_matrix['revenue'].sort_values(ascending=False )
train_df.columns.tolist()<import_modules>
from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, PowerTransformer, MultiLabelBinarizer
from sklearn.ensemble import RandomForestClassifier
import xgboost as xgb<train_model>
X = data.drop(columns=['revenue'], axis=1 ).copy()
y = data['revenue'].copy()
X.shape, y.shape
clf = RandomForestClassifier()
clf.fit(x, y)
 imp = clf.feature_importances_
name = np.array(x.columns.values.tolist())
 df_imp = pd.DataFrame({'feature':name, 'importance':imp})
 df_imp = df_imp.sort_values(by='importance', ascending=False )<features_selection>
top_30_values = dict()
list_of_genres_names = list(X['genres'].apply(lambda x: [] if pd.isna(x)else ast.literal_eval(x)).apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 top_30_genres =(Counter([i for j in list_of_genres_names for i in j] ).most_common(30))
top_30_values['genres'] = [x for x, y in top_30_genres]
list_of_production_companies_names = list(X['production_companies'].apply(lambda x: [] if pd.isna(x)else ast.literal_eval(x)).apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 top_30_production_companies =(Counter([i for j in list_of_production_companies_names for i in j] ).most_common(30))
top_30_values['production_companies'] = [x for x, y in top_30_production_companies]
list_of_production_countries_names = list(X['production_countries'].apply(lambda x: [] if pd.isna(x)else ast.literal_eval(x)).apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 top_30_production_countries =(Counter([i for j in list_of_production_countries_names for i in j] ).most_common(30))
top_30_values['production_countries'] = [x for x, y in top_30_production_countries]
list_of_keywords = list(X['Keywords'].apply(lambda x: [] if pd.isna(x)else ast.literal_eval(x)).apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 top_30_keywords =(Counter([i for j in list_of_keywords for i in j] ).most_common(30))
top_30_values['Keywords'] = [x for x, y in top_30_keywords]
list_of_cast_names = list(X['cast'].apply(lambda x: [] if pd.isna(x)else ast.literal_eval(x)).apply(lambda x: [i['name'] for i in x if i['order'] in(0, 1, 2)] if x != {} else [] ).values)
 top_30_cast =(Counter([i for j in list_of_cast_names for i in j] ).most_common(30))
top_30_values['cast'] = [x for x, y in top_30_cast]
def feature_importance(forest, X_train):
ranked_list = []
importances = forest.feature_importances_
indices = np.argsort(importances)[::-1]
print("Feature ranking:")
 for f in range(X_train.shape[1]):
print("%d.feature %d(%f)" %(f + 1, indices[f], importances[indices[f]])+ " - " + X_train.columns[indices[f]])
 ranked_list.append(X_train.columns[indices[f]])
 return ranked_list<filter>
drop_cols = ['id', 'homepage', 'imdb_id', 'original_title', 'spoken_languages', 'overview', 'poster_path', 'tagline', 'title', 'crew']
encoded_cols = []
important_cols = df_imp.loc[(df_imp['importance']>0),'feature']<import_modules>
class CustomAttr(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X):
try:
X['is_sequel'] = X['belongs_to_collection'].apply(lambda x: 0 if pd.isna(x)else 1)
 X['release_date']= X['release_date'].apply(lambda x: self.modify_date(x))
X['release_year'] = pd.DatetimeIndex(X['release_date'] ).year
X['release_month'] = pd.DatetimeIndex(X['release_date'] ).month
X['release_day'] = pd.DatetimeIndex(X['release_date'] ).day
X['release_dow'] = pd.DatetimeIndex(X['release_date'] ).dayofweek
X = X.drop(['belongs_to_collection', 'release_date'], axis=1)
 return X
except Exception as e:
print("CustomAttr: Exception caught: {}".format(e))
@staticmethod
def modify_date(x):

 try:
if x is np.nan:
x='01/01/00'
x=str(x)
 year=x.split('/')[2]
if int(year)<20:
return x[:-2]+'20'+year
else:
return x[:-2]+'19'+year
except Exception as e:
print("CustomAttr: modify_date() function - exception caught for date {}: {}".format(x,e))
from sklearn.preprocessing import StandardScaler<define_variables>
class JSONHandler(BaseEstimator, TransformerMixin):
def __init__(self):

 self.mlbs = dict()
def fit(self, X, y=None):
return self
def transform(self, X):
for col in list(X.columns):
try:
X[col] = X[col].apply(lambda x: [] if pd.isna(x)else ast.literal_eval(x))
X[col] = X[col].apply(lambda x: self.get_names(x, col))
if not(col in self.mlbs.keys()):
self.mlbs[col] = MultiLabelBinarizer()
X_enc = pd.DataFrame(self.mlbs[col].fit_transform(X[col]),columns=self.mlbs[col].classes_,
index=X.index)
 encoded_cols.extend(list(self.mlbs[col].classes_))
else:
X_enc = pd.DataFrame(self.mlbs[col].transform(X[col]),columns=self.mlbs[col].classes_,
index=X.index)
 X = X.drop(col, axis=1)
 X = pd.concat([X, X_enc], axis=1)
 except Exception as e:
print("JSONHandler: Exception caught for {}: {}".format(col,e))
return X
@staticmethod
def get_names(x, col):

 names = []
try:
names = [item['name'] for item in x if item['name'] in top_30_values[col]]
if len(names)==0:
names.append('other_'+col)
 return names
except Exception as e:
print("JSONHandler: get_names() function - exception caught {}: {}".format(x,e))
features = [c for c in x_.columns if c not in ['Target']]
target = train_df['Target'] 
<import_modules>
num_pipeline = Pipeline([('imputer', SimpleImputer(strategy='median')) , 
('scaler', PowerTransformer())])
 cat_pipeline = Pipeline([('imputer', SimpleImputer(strategy='most_frequent')) , 
('cat_enc', OneHotEncoder(handle_unknown='ignore')) ])
 pre_process = ColumnTransformer([('drop_cols', 'drop', drop_cols), 
('num_process', num_pipeline, ['budget', 'popularity', 'runtime']), 
('add_custom_attrs', CustomAttr() , ['belongs_to_collection', 'release_date']), 
('cat_process', cat_pipeline, ['original_language', 'status']), 
('jason_handler', JSONHandler() , ['genres', 'production_companies', 'production_countries', 'Keywords', 'cast'])], remainder='passthrough')
 X_transformed = pre_process.fit_transform(X)
 X_transformed.shape
from sklearn.model_selection import train_test_split<import_modules>
X_train_transformed, X_test_transformed, y_train, y_test = train_test_split(X_transformed, np.log1p(y), test_size=0.2, random_state=42)
 X_train_transformed.shape, X_test_transformed.shape
from sklearn.model_selection import train_test_split<train_model>
feature_columns = ['budget', 'popularity', 'runtime', 'is_sequel', 'release_year', 'release_month', 'release_day', 'release_dow'] + list(pre_process.transformers_[3][1]['cat_enc'].get_feature_names(['original_language', 'status'])) + encoded_cols
len(feature_columns )
smote_tomek = SMOTETomek(random_state=0)
 X_resampled, y_resampled = smote_tomek.fit_sample(x_, target )<train_model>
kf = KFold(n_splits=5, shuffle=True, random_state=42 )
<split>
results = []
def performance_measures(model, store_results=True):
train_rmses = cross_val_score(model, X_train_transformed, y_train, scoring='neg_root_mean_squared_error', cv=kf, n_jobs=-1)
 train_rmses *= -1
train_mean_rmse = np.mean(train_rmses)
 test_rmses = cross_val_score(model, X_test_transformed, y_test, scoring='neg_root_mean_squared_error', cv=kf, n_jobs=-1)
 test_rmses *= -1
test_mean_rmse = np.mean(test_rmses)
 print("Train Mean RMSE: {}
Test Mean RMSE: {}".format(train_mean_rmse, test_mean_rmse))
if store_results:
results.append([model.__class__.__name__, train_mean_rmse, test_mean_rmse] )
X_train, X_valid, y_train, y_valid = train_test_split(X_resampled, y_resampled, test_size=0.1, random_state=1)
<create_dataframe>
ridge_reg = Ridge(alpha=1, random_state=42)
 ridge_reg.fit(X_train_transformed, y_train )
pd.options.display.width =300
X_resampled = pd.DataFrame(X_resampled)
 X_resampled.columns = features
y_resampled = pd.DataFrame(y_resampled)
 y_resampled.columns = ['Target']
X_train_df = pd.DataFrame(X_train)
 X_train_df.columns = features<import_modules>
performance_measures(ridge_reg )
import xgboost as xgb
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold
from sklearn.utils.class_weight import compute_class_weight, compute_sample_weight
from sklearn.metrics import f1_score
from sklearn.ensemble import VotingClassifier<choose_model_class>
forest_reg = RandomForestRegressor(n_estimators=500, max_depth=16, max_features=0.2, n_jobs=-1, random_state=42)
 forest_reg.fit(X_train_transformed, y_train )
skf = StratifiedKFold(n_splits=5, shuffle=True )<choose_model_class>
performance_measures(forest_reg )
lgbm = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=None, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced',
colsample_bytree = 0.88, min_child_samples = 90, num_leaves = 16, subsample = 0.94 )<predict_on_test>
xgb_reg = XGBRegressor(objective='reg:squarederror', n_estimators = 1000, max_depth = 14, learning_rate = 0.01,
gamma=1.0, subsample = 0.7, colsample_bytree = 0.6, colsample_bylevel = 0.5,
random_state=42, n_jobs=-1)
 xgb_reg.fit(X_train_transformed, y_train )
predicts_lgb = []
for train_index, test_index in skf.split(X_resampled, y_resampled):
X_t, X_v = X_resampled.iloc[train_index], X_resampled.iloc[test_index]
y_t, y_v = y_resampled.iloc[train_index], y_resampled.iloc[test_index]
lgbm.fit(X_t, y_t)
 predicts_lgb.append(lgbm.predict(test_df[features]))<train_model>
performance_measures(xgb_reg )
<create_dataframe>
cat_boost_reg = CatBoostRegressor(loss_function='RMSE', bagging_temperature = 0.3, colsample_bylevel = 0.7,
depth = 9, eval_metric = 'RMSE', iterations = 1500,
random_state=42, verbose=0)
 cat_boost_reg.fit(X_train_transformed, y_train )
submission = pd.DataFrame()
submission['Id'] = test_df['Id']
submission['Target'] = np.array(predicts_lgb ).mean(axis=0 ).round().astype(int)
<save_to_csv>
performance_measures(cat_boost_reg )
submission.to_csv('submissions.csv', index=False )<set_options>
lgb_reg = LGBMRegressor(objective = 'regression', num_iterations = 100, max_depth = 12, learning_rate= 0.03,
metric = 'rmse', colsample_bytree= 0.6, subsample_freq= 1, subsample= 0.5, n_jobs=-1,
random_state=42)
 lgb_reg.fit(X_train_transformed, y_train )
warnings.filterwarnings('ignore' )<init_hyperparams>
performance_measures(lgb_reg )
n_jobs = multiprocessing.cpu_count()
n_jobs<import_modules>
named_estimators = [('cat_bost', cat_boost_reg),('xgb_reg', xgb_reg),('lgb_reg', lgb_reg),('forest_reg', forest_reg),('ridge_reg', ridge_reg)]
voting_reg = VotingRegressor(estimators=named_estimators, n_jobs=-1)
 voting_reg.fit(X_train_transformed, y_train )
from sklearn.metrics import classification_report
from sklearn.feature_selection import SelectFromModel
from sklearn.model_selection import KFold, GridSearchCV, cross_val_score, cross_val_predict
from sklearn.metrics import accuracy_score, recall_score, f1_score
from sklearn.metrics.scorer import make_scorer
from xgboost import plot_tree, plot_importance 
<import_modules>
performance_measures(voting_reg )
import xgboost as xgb
from sklearn.model_selection import train_test_split, cross_validate
from sklearn.feature_selection import VarianceThreshold, SelectKBest, f_classif
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix<set_options>
pd.DataFrame(results, columns=['Model', 'Train RMSE', 'Test RMSE'] )
%matplotlib inline<load_from_csv>
predicted_revenue = voting_reg.predict(X_transformed)
 overall_data = X.copy()
overall_data['revenue'] = y.copy()
overall_data['predicted_revenue'] = np.expm1(predicted_revenue )
df = pd.read_csv('.. /input/train.csv' )<set_options>
final_model = Pipeline([('pre_process', pre_process), 
('voting_reg', voting_reg)])
 final_model.fit(X, np.log1p(y))
pd.options.display.max_columns = num_of_cols<create_dataframe>
test_data = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 print(test_data.shape)
 test_data.head(n=3 )
df_isna = pd.DataFrame(df.isnull().sum())
 df_isna.loc[(df_isna.loc[:, df_isna.dtypes != object] != 0 ).any(1)]<groupby>
test_data.isna().sum()
hid_heads = df.groupby(['idhogar'])['parentesco1'].apply(lambda x: pd.unique(x.values.ravel() ).tolist() ).reset_index()
len(hid_heads )<prepare_output>
predictions = final_model.predict(test_data)
 predictions = np.expm1(predictions )
df_hid = pd.DataFrame(hid_heads, index=None, columns=['idhogar','parentesco1'])
 df_hid.sample(5 )<feature_engineering>
output = pd.DataFrame(test_data['id'])
 output['revenue'] = predictions.copy()
df_hid['parentesco1'] = df_hid['parentesco1'].apply(lambda x: ''.join(map(str, x)) )<filter>
output.to_csv("./submission.csv", index=False )