Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<sort_values>
plt.style.use('ggplot')
 %config InlineBackend.figure_format = 'svg'
sorted(num_cols )<define_variables>
!pip install bert-serving-server
!pip install bert-serving-client
cols_electronics = ['refrig','mobilephone','television','qmobilephone','computer', 'v18q', 'v18q1', ]
cols_house_details = ['v2a1', 'area1', 'area2', 'bedrooms','rooms', 'cielorazo', 'v14a',
'tamhog', 'hacdor', 'hacapo', 'r4t3', ]
cols_person_details = ['age', 'agesq', 'female', 'male',]
cols_SQ = ['SQBage', 'SQBdependency', 'SQBedjefe', 'SQBescolari', 'SQBhogar_nin',
'SQBhogar_total', 'SQBmeaned', 'SQBovercrowding',]
cols_water = ['abastaguadentro', 'abastaguafuera', 'abastaguano',]
cols_h = [ 'hhsize', 'hogar_adul', 'hogar_mayor', 'hogar_nin', 'hogar_total',]
cols_r = ['r4h1', 'r4h2', 'r4h3', 'r4m1', 'r4m2', 'r4m3', 'r4t1', 'r4t2', 'r4t3',]
cols_tip = ['tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',]
cols_roof = ['techocane', 'techoentrepiso', 'techootro', 'techozinc',]
cols_floor = ['pisocemento', 'pisomadera', 'pisomoscer', 'pisonatur', 'pisonotiene', 'pisoother',]
cols_sanitary = [ 'sanitario1', 'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',]
cols_parents = [ 'parentesco1', 'parentesco10', 'parentesco11', 'parentesco12', 'parentesco2', 'parentesco3',
'parentesco4', 'parentesco5', 'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9',]
cols_outside_wall = [ 'paredblolad', 'pareddes', 'paredfibras', 'paredmad', 'paredother',
'paredpreb', 'paredzinc', 'paredzocalo',]
cols_instlevel = [ 'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6',
'instlevel7', 'instlevel8', 'instlevel9',]
cols_lugar = [ 'lugar1', 'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6',]
cols_estadoc = [ 'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4',
'estadocivil5', 'estadocivil6', 'estadocivil7',]
cols_elim = ['elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4', 'elimbasu5', 'elimbasu6',]
cols_energ = ['energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',]
cols_eviv = [ 'eviv1', 'eviv2', 'eviv3',]
cols_etech = [ 'etecho1', 'etecho2', 'etecho3',]
cols_pared = [ 'epared1', 'epared2', 'epared3',]
cols_unknown = [ 'dis', 'escolari', 'meaneduc',
'overcrowding', 'rez_esc', 'tamhog', 'tamviv', ]
cols_elec = ['coopele', 'noelec', 'planpri', 'public',]
total_features = cols_electronics+cols_house_details+cols_person_details+\
cols_SQ+cols_water+cols_h+cols_r+cols_tip+cols_roof+\
cols_floor+cols_sanitary+cols_parents+cols_outside_wall+\
cols_instlevel+cols_lugar+cols_estadoc+cols_elim+cols_energ+\
cols_eviv+cols_etech+cols_pared+cols_unknown+cols_elec
len(total_features )<drop_column>
!pip install git+http://github.com/brendanhasz/dsutils.git
cols_electronics.remove('Target')
 cols_electronics<drop_column>
dtypes = {
'id': 'uint16',
'belongs_to_collection': 'str',
'budget': 'float32',
'genres': 'str',
'homepage': 'str',
'imdb_id': 'str',
'original_language': 'str',
'overview': 'str',
'popularity': 'float32',
'production_companies': 'str',
'production_countries': 'str',
'release_date': 'str',
'runtime': 'float32',
'spoken_languages': 'str',
'tagline': 'str',
'title': 'str',
'Keywords': 'str',
'cast': 'str',
'crew': 'str',
'revenue': 'float32',
}
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv',
usecols=dtypes.keys() ,
dtype=dtypes)
 del dtypes['revenue']
test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv',
usecols=dtypes.keys() ,
dtype=dtypes)
 df = pd.concat([train, test], axis=0 )
total_features.remove('r4t3')
 total_features.remove('tamhog')
 total_features.remove('tamviv')
 len(total_features )<define_variables>
dtypes = {
'imdb_id': 'str',
'avg_rating': 'float32',
'num_rating': 'float32',
}
imdb_df = pd.read_csv('.. /input/box-office-prediction-imbd-scores/imdb_scores.csv',
usecols=dtypes.keys() ,
dtype=dtypes )
for x,y in zip(num_cols, list(es_corr)) :
if(y >= 0.75)or(y < -0.6):
print(x,y )<define_variables>
def fix_dates(date_str):
if isinstance(date_str, str):
if int(date_str[-2:])< 20:
return date_str[:-2]+'20'+date_str[-2:]
else:
return date_str[:-2]+'19'+date_str[-2:]
else:
return np.nan
train['release_date'] = train['release_date'].apply(fix_dates)
 test['release_date'] = test['release_date'].apply(fix_dates )
for x,y in zip(num_cols, list(sqbes_corr)) :
if(y >= 0.5)or(y < -0.6):
print(x,y )<drop_column>
df.loc[df['runtime']==0, :]
total_features.remove('escolari')
 len(total_features )<data_type_conversions>
train.loc[train['runtime']<1, 'runtime'] = np.nan
test.loc[test['runtime']<1, 'runtime'] = np.nan
df['meaneduc'].fillna(0, inplace=True)
 df['SQBmeaned'].fillna(0, inplace=True )<drop_column>
train.set_index('id', inplace=True)
 test.set_index('id', inplace=True )
total_features.remove('meaneduc')
 total_features<categorify>
train_y = train['revenue']
train_X = train
del train_X['revenue']
df['v18q1'] = df['v18q'].groupby(df['idhogar'] ).transform('sum' )<create_dataframe>
train_y = np.log1p(train_y )
ff = pd.DataFrame(df.isnull().sum())
 ff.loc[(ff.loc[:, ff.dtypes != object] != 0 ).any(1)]<count_missing_values>
transforms = {
'budget': lambda x: np.log1p(x),
'popularity': lambda x: np.log1p(x),
}
null_encode_cols = [
'belongs_to_collection',
'homepage',
]
date_cols = {
'release_date':('%m/%d/%Y', ['year', 'month', 'day', 'dayofyear', 'dayofweek'])
 }
json_fields = {
'genres': 'name',
'production_companies': 'name',
'production_countries': 'iso_3166_1',
'spoken_languages': 'iso_639_1',
'Keywords': 'name',
'cast': 'name',
'crew': [('name', 'job', 'Director'), 
('name', 'job', 'Producer'), 
('name', 'job', 'Writer'), 
('name', 'job', 'Editor'),],
}
nhot_cols = [
'genres_name',
'original_language',
'production_countries_iso_3166_1',
'spoken_languages_iso_639_1',
]
bayesian_c = 5
te_cols = [
'production_companies_name',
'cast_name',
'crew_job_Director_name',
'crew_job_Producer_name',
'crew_job_Writer_name',
'crew_job_Editor_name',
]
n_pc = 5
bert_cols = [
'overview',
'tagline',
'title',
'Keywords_name',
]
word_count = lambda e: len(e.split(' ')) if isinstance(e, str)else 0
keyword_count = lambda e: len(e.split(',')) if isinstance(e, str)else 0
new_features = {
'budget_runtime_ratio': lambda x: x['budget']/x['runtime'],
'budget_popularity_ratio': lambda x: x['budget']/(x['popularity']+1),
'budget_year_ratio': lambda x: x['budget']/np.square(x['release_date_year']),
'popularity_year_ratio': lambda x: x['popularity']/np.square(x['release_date_year']),
'rating_to_votes_ratio': lambda x: x['avg_rating']/(x['num_rating']+1),
'runtime_rating_ratio': lambda x: x['runtime']/(x['avg_rating']+1),
'overview_word_count': lambda x: x['overview'].apply(word_count),
'tagline_word_count': lambda x: x['tagline'].apply(word_count),
'keyword_count': lambda x: x['Keywords_name'].apply(keyword_count),
}
df['rez_esc'].isnull().sum()<count_missing_values>
preprocessing = Pipeline([ 
('transforms', LambdaTransformer(transforms)) , 
('join_imbd', JoinTransformer(imdb_df, 'imdb_id', 'imdb_id')) , 
('null_encoder', NullEncoder(null_encode_cols, delete_old=True)) , 
('date_encoder', DateEncoder(date_cols)) , 
('json_encoder', JsonEncoder(json_fields)) , 
('nhot_encoder', NhotEncoder(nhot_cols, top_n=10)) , 
('add_features', LambdaFeatures(new_features)) , 
('targ_encoder', MultiTargetEncoderLOO(te_cols, bayesian_c=bayesian_c)) , 
('bert_encoder', BertEncoder(bert_cols, n_pc=n_pc)) , 
('scaler', Scaler()), 
('imputer', Imputer()),
] )
len(df)- df['rez_esc'].isnull().sum()<count_missing_values>

 model = Pipeline([ 
('preprocessing', preprocessing), 
('regressor', CatBoostRegressor(verbose=False))
])
 fit_model = model.fit(train_X, train_y)
 preds = fit_model.predict(test)
 preds_df = pd.DataFrame(index=test.index)
 preds_df['revenue'] = np.maximum(0, np.expm1(preds))
preds_df.to_csv('predictions_baseline.csv' )
df['v2a1'].isnull().sum()<count_missing_values>
train_pp = preprocessing.fit_transform(train_X, train_y)
 test_pp = preprocessing.transform(test )
len(df)- df['v2a1'].isnull().sum()<filter>
%%time
catboost = Pipeline([ 
('regressor', CatBoostRegressor(verbose=False))
])
 imp_df = permutation_importance_cv(train_pp,
train_y.copy() ,
estimator=catboost,
metric='rmse',
n_jobs=1 )
len(df.loc[(df.v2a1 >= 0)]), len(df.loc[(df.rez_esc >= 0)] )<filter>
cols_to_keep = top_k_permutation_importances(imp_df, k=30 )
len(df.loc[(df.v2a1 >= 0)&(df.rez_esc >= 0)] )<filter>
preprocessing = Pipeline([ 
('preprocessing', preprocessing), 
('col_filter', KeepOnlyCols(cols_to_keep)) ,
] )
len(df.loc[(df.v2a1 >= 0)|(df.rez_esc >= 0)] )<correct_missing_values>
base_learners = [
BayesianRidge() ,
XGBRegressor() ,
CatBoostRegressor(verbose=False),
LGBMRegressor()
]
model = StackedRegressor(base_learners,
meta_learner=BayesianRidge() ,
preprocessing=preprocessing,
n_splits=5, n_jobs=1 )
df['rez_esc'].fillna(0, inplace=True )<create_dataframe>
%%time
fit_model = model.fit(train_X, train_y)
 preds = fit_model.predict(test)
 preds_df = pd.DataFrame(index=test.index)
 preds_df['revenue'] = np.maximum(0, np.expm1(preds))
preds_df.to_csv('predictions.csv' )