Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<data_type_conversions>
py.init_notebook_mode(connected=True)
 pd.set_option('max_columns', None)
 warnings.filterwarnings("ignore" )
bureau.AMT_CREDIT_SUM_DEBT.fillna(0.0,inplace=True )<drop_column>
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 dict_columns = ['belongs_to_collection','genres','spoken_languages','production_companies',
'production_countries','Keywords','cast','crew']
def text_to_dict(df):
for columns in dict_columns:
df[columns] = df[columns].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
train = text_to_dict(train)
 test = text_to_dict(test)
 test['revenue'] = np.nan
train = pd.merge(train, pd.read_csv('.. /input/tmdb-competition-additional-features/TrainAdditionalFeatures.csv'), how='left', on=['imdb_id'])
 test = pd.merge(test, pd.read_csv('.. /input/tmdb-competition-additional-features/TestAdditionalFeatures.csv'), how='left', on=['imdb_id'])
 train.head(2 )
bureau.drop('AMT_ANNUITY',axis=1,inplace=True )<categorify>
train.loc[train['id'] == 16,'revenue'] = 192864
train.loc[train['id'] == 90,'budget'] = 30000000
train.loc[train['id'] == 118,'budget'] = 60000000
train.loc[train['id'] == 149,'budget'] = 18000000
train.loc[train['id'] == 313,'revenue'] = 12000000
train.loc[train['id'] == 451,'revenue'] = 12000000
train.loc[train['id'] == 464,'budget'] = 20000000
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 930000
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 819,'budget'] = 90000000
train.loc[train['id'] == 850,'budget'] = 90000000
train.loc[train['id'] == 1007,'budget'] = 2
train.loc[train['id'] == 1112,'budget'] = 7500000
train.loc[train['id'] == 1131,'budget'] = 4300000
train.loc[train['id'] == 1359,'budget'] = 10000000
train.loc[train['id'] == 1542,'budget'] = 1
train.loc[train['id'] == 1570,'budget'] = 15800000
train.loc[train['id'] == 1571,'budget'] = 4000000
train.loc[train['id'] == 1714,'budget'] = 46000000
train.loc[train['id'] == 1721,'budget'] = 17500000
train.loc[train['id'] == 1865,'revenue'] = 25000000
train.loc[train['id'] == 1885,'budget'] = 12
train.loc[train['id'] == 2091,'budget'] = 10
train.loc[train['id'] == 2268,'budget'] = 17500000
train.loc[train['id'] == 2491,'budget'] = 6
train.loc[train['id'] == 2602,'budget'] = 31000000
train.loc[train['id'] == 2612,'budget'] = 15000000
train.loc[train['id'] == 2696,'budget'] = 10000000
train.loc[train['id'] == 2801,'budget'] = 10000000
train.loc[train['id'] == 335,'budget'] = 2
train.loc[train['id'] == 348,'budget'] = 12
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 1100000
train.loc[train['id'] == 640,'budget'] = 6
train.loc[train['id'] == 696,'budget'] = 1
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 850,'budget'] = 1500000
train.loc[train['id'] == 1199,'budget'] = 5
train.loc[train['id'] == 1282,'budget'] = 9
train.loc[train['id'] == 1347,'budget'] = 1
train.loc[train['id'] == 1755,'budget'] = 2
train.loc[train['id'] == 1801,'budget'] = 5
train.loc[train['id'] == 1918,'budget'] = 592
train.loc[train['id'] == 2033,'budget'] = 4
train.loc[train['id'] == 2118,'budget'] = 344
train.loc[train['id'] == 2252,'budget'] = 130
train.loc[train['id'] == 2256,'budget'] = 1
train.loc[train['id'] == 2696,'budget'] = 10000000
test.loc[test['id'] == 6733,'budget'] = 5000000
test.loc[test['id'] == 3889,'budget'] = 15000000
test.loc[test['id'] == 6683,'budget'] = 50000000
test.loc[test['id'] == 5704,'budget'] = 4300000
test.loc[test['id'] == 6109,'budget'] = 281756
test.loc[test['id'] == 7242,'budget'] = 10000000
test.loc[test['id'] == 7021,'budget'] = 17540562
test.loc[test['id'] == 5591,'budget'] = 4000000
test.loc[test['id'] == 4282,'budget'] = 20000000
test.loc[test['id'] == 3033,'budget'] = 250
test.loc[test['id'] == 3051,'budget'] = 50
test.loc[test['id'] == 3084,'budget'] = 337
test.loc[test['id'] == 3224,'budget'] = 4
test.loc[test['id'] == 3594,'budget'] = 25
test.loc[test['id'] == 3619,'budget'] = 500
test.loc[test['id'] == 3831,'budget'] = 3
test.loc[test['id'] == 3935,'budget'] = 500
test.loc[test['id'] == 4049,'budget'] = 995946
test.loc[test['id'] == 4424,'budget'] = 3
test.loc[test['id'] == 4460,'budget'] = 8
test.loc[test['id'] == 4555,'budget'] = 1200000
test.loc[test['id'] == 4624,'budget'] = 30
test.loc[test['id'] == 4645,'budget'] = 500
test.loc[test['id'] == 4709,'budget'] = 450
test.loc[test['id'] == 4839,'budget'] = 7
test.loc[test['id'] == 3125,'budget'] = 25
test.loc[test['id'] == 3142,'budget'] = 1
test.loc[test['id'] == 3201,'budget'] = 450
test.loc[test['id'] == 3222,'budget'] = 6
test.loc[test['id'] == 3545,'budget'] = 38
test.loc[test['id'] == 3670,'budget'] = 18
test.loc[test['id'] == 3792,'budget'] = 19
test.loc[test['id'] == 3881,'budget'] = 7
test.loc[test['id'] == 3969,'budget'] = 400
test.loc[test['id'] == 4196,'budget'] = 6
test.loc[test['id'] == 4221,'budget'] = 11
test.loc[test['id'] == 4222,'budget'] = 500
test.loc[test['id'] == 4285,'budget'] = 11
test.loc[test['id'] == 4319,'budget'] = 1
test.loc[test['id'] == 4639,'budget'] = 10
test.loc[test['id'] == 4719,'budget'] = 45
test.loc[test['id'] == 4822,'budget'] = 22
test.loc[test['id'] == 4829,'budget'] = 20
test.loc[test['id'] == 4969,'budget'] = 20
test.loc[test['id'] == 5021,'budget'] = 40
test.loc[test['id'] == 5035,'budget'] = 1
test.loc[test['id'] == 5063,'budget'] = 14
test.loc[test['id'] == 5119,'budget'] = 2
test.loc[test['id'] == 5214,'budget'] = 30
test.loc[test['id'] == 5221,'budget'] = 50
test.loc[test['id'] == 4903,'budget'] = 15
test.loc[test['id'] == 4983,'budget'] = 3
test.loc[test['id'] == 5102,'budget'] = 28
test.loc[test['id'] == 5217,'budget'] = 75
test.loc[test['id'] == 5224,'budget'] = 3
test.loc[test['id'] == 5469,'budget'] = 20
test.loc[test['id'] == 5840,'budget'] = 1
test.loc[test['id'] == 5960,'budget'] = 30
test.loc[test['id'] == 6506,'budget'] = 11
test.loc[test['id'] == 6553,'budget'] = 280
test.loc[test['id'] == 6561,'budget'] = 7
test.loc[test['id'] == 6582,'budget'] = 218
test.loc[test['id'] == 6638,'budget'] = 5
test.loc[test['id'] == 6749,'budget'] = 8
test.loc[test['id'] == 6759,'budget'] = 50
test.loc[test['id'] == 6856,'budget'] = 10
test.loc[test['id'] == 6858,'budget'] = 100
test.loc[test['id'] == 6876,'budget'] = 250
test.loc[test['id'] == 6972,'budget'] = 1
test.loc[test['id'] == 7079,'budget'] = 8000000
test.loc[test['id'] == 7150,'budget'] = 118
test.loc[test['id'] == 6506,'budget'] = 118
test.loc[test['id'] == 7225,'budget'] = 6
test.loc[test['id'] == 7231,'budget'] = 85
test.loc[test['id'] == 5222,'budget'] = 5
test.loc[test['id'] == 5322,'budget'] = 90
test.loc[test['id'] == 5350,'budget'] = 70
test.loc[test['id'] == 5378,'budget'] = 10
test.loc[test['id'] == 5545,'budget'] = 80
test.loc[test['id'] == 5810,'budget'] = 8
test.loc[test['id'] == 5926,'budget'] = 300
test.loc[test['id'] == 5927,'budget'] = 4
test.loc[test['id'] == 5986,'budget'] = 1
test.loc[test['id'] == 6053,'budget'] = 20
test.loc[test['id'] == 6104,'budget'] = 1
test.loc[test['id'] == 6130,'budget'] = 30
test.loc[test['id'] == 6301,'budget'] = 150
test.loc[test['id'] == 6276,'budget'] = 100
test.loc[test['id'] == 6473,'budget'] = 100
test.loc[test['id'] == 6842,'budget'] = 30
bureau_onehot=onehot_encoding(bureau,categorical_features(bureau))
bureau_onehot.head()<set_options>
train['popularity'] = np.log1p(train['popularity'])
 train['revenue'] = np.log1p(train['revenue'])
 train['totalVotes'] = np.log1p(train['totalVotes'])
 train['budget'] = np.log1p(train['budget'])
 train['runtime'] = np.log1p(train['runtime'])
 train['popularity2'] = np.log1p(train['popularity2'])
 test['popularity'] = np.log1p(test['popularity'])
 test['totalVotes'] = np.log1p(test['totalVotes'])
 test['budget'] = np.log1p(test['budget'])
 test['runtime'] = np.log1p(test['runtime'])
 test['popularity2'] = np.log1p(test['popularity2'] )
del bureau
gc.collect()<groupby>
for i,e in enumerate(train['belongs_to_collection'][:2]):
print(i,e )
month_count=bureau_balance.groupby('SK_ID_BUREAU' ).size()<count_values>
train['belongs_to_collection'].apply(lambda x: 1 if x!= {} else 0 ).value_counts()
bureau_balance.STATUS.value_counts()<data_type_conversions>
train['has_collection'] = train['belongs_to_collection'].apply(lambda x: len(x)if x!={} else 0)
 test['has_collection'] = test['belongs_to_collection'].apply(lambda x: len(x)if x!={} else 0 )
bureau_balance_unstack=bureau_balance.groupby('SK_ID_BUREAU')['STATUS'].value_counts(normalize = False ).unstack('STATUS')
 bureau_balance_unstack.columns=['status_DPD0','status_DPD1','status_DPD2','status_DPD3','status_DPD4','status_DPD5','status_closed','status_X']
bureau_balance_unstack['month_count']=month_count
bureau_balance_unstack.fillna(value=0,inplace=True)
 bureau_balance_unstack.head()<set_options>
for i,e in enumerate(train['genres'][:2]):
print(i,e )
del bureau_balance
gc.collect()<merge>
print('Number of genres in films:')
 train['genres'].apply(lambda x: len(x)if x!={} else 0 ).value_counts()
bureau_merge=bureau_onehot.merge(bureau_balance_unstack,how='left',on='SK_ID_BUREAU' )<groupby>
list_of_genres = list(train['genres'].apply(lambda x: [i['name'] for i in x] if x!={} else [] ).values )
cnt_id_bureau=bureau_merge[['SK_ID_CURR','SK_ID_BUREAU']].groupby('SK_ID_CURR' ).size()<drop_column>
train['num_of_genres'] = train['genres'].apply(lambda x: len(x)if x!={} else 0)
 train['all_genres'] = train['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x ]))
if x!= {} else '')
 test['num_of_genres'] = test['genres'].apply(lambda x: len(x)if x!={} else 0)
 test['all_genres'] = test['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x ]))
if x!= {} else '' )
del bureau_merge,bureau_onehot,bureau_balance_unstack
gc.collect()<feature_engineering>
for g in top_genres:
train['genre_' + g] = train['all_genres'].apply(lambda x: 1 if g in x else 0)
 test['genre_' + g] = test['all_genres'].apply(lambda x: 1 if g in x else 0 )
categorical_features(previous_application )<drop_column>
for i,e in enumerate(train['production_companies'][:2]):
print(i,e )
previous_application.drop(['RATE_INTEREST_PRIVILEGED','RATE_INTEREST_PRIMARY'],axis=1,inplace=True )<data_type_conversions>
print('Number of Production Companies for a movie:')
 train['production_companies'].apply(lambda x: len(x)if x!= {} else 0 ).value_counts()
previous_application.AMT_CREDIT.fillna(previous_application.AMT_CREDIT.median() ,inplace=True )<count_values>
train[train['production_companies'].apply(lambda x: len(x)if x!= {} else 0)> 10]
previous_application.CHANNEL_TYPE.value_counts()<drop_column>
list_of_companies = list(train['production_companies'].apply(lambda x : [i['name'] for i in x]
if x!= {} else [] ).values)
 Counter(i for j in list_of_companies for i in j ).most_common(20 )
previous_application.drop(['PRODUCT_COMBINATION','NAME_TYPE_SUITE',],axis=1,inplace=True )<data_type_conversions>
train['num_prod_companies'] = train['production_companies'].apply(lambda x: len(x)if
x!={} else 0)
 test['num_prod_companies'] = test['production_companies'].apply(lambda x: len(x)if
x!={} else 0)
 train['all_prod_companies'] = train['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x!={} else '')
 test['all_prod_companies'] = test['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x!={} else '' )
previous_application.RATE_DOWN_PAYMENT.fillna(previous_application.RATE_DOWN_PAYMENT.median() ,inplace=True )<data_type_conversions>
top_prod_companies = [m[0] for m in Counter(i for j in list_of_companies for i in j ).most_common(10)]
for pc in top_prod_companies:
train['production_' + pc] = train['all_prod_companies'].apply(lambda x: 1 if pc in x else 0)
 test['production_'+ pc] = test['all_prod_companies'].apply(lambda x: 1 if pc in x else 0 )
previous_application.AMT_DOWN_PAYMENT.fillna(0.0,inplace=True )<data_type_conversions>
for i, e in enumerate(train['production_countries'][:2]):
print(i,e )
previous_application.AMT_GOODS_PRICE.fillna(previous_application.AMT_GOODS_PRICE.mean() ,inplace=True )<data_type_conversions>
print('Number of Production Countries in Movies:')
 train['production_countries'].apply(lambda x: len(x)if x!={} else 0 ).value_counts()
previous_application.AMT_ANNUITY.fillna(previous_application.AMT_ANNUITY.mean() ,inplace=True )<data_type_conversions>
train[train['production_countries'].apply(lambda x: len(x)if x!= {} else 0)> 5]
previous_application.CNT_PAYMENT.fillna(previous_application.CNT_PAYMENT.median() ,inplace=True )<categorify>
List_of_countries = list(train['production_countries'].apply(lambda x: [i['name'] for i in x]
if x!= {} else []))
Counter(i for j in List_of_countries for i in j ).most_common(10 )
previous_application_onehot=onehot_encoding(previous_application,categorical_features(previous_application))<groupby>
train['num_prod_countries'] = train['production_countries'].apply(lambda x: len(x)if x!= {}
else 0)
 test['num_prod_countries'] = test['production_countries'].apply(lambda x: len(x)if x!={}
else 0)
 train['all_prod_countries'] = train['production_countries'].apply(lambda x: ' '.join(sorted(i['name'] for i in x))
if x!= {} else '')
 test['all_prod_countries'] = test['production_countries'].apply(lambda x: ' '.join(sorted(i['name'] for i in x))
if x!= {} else '')
cnt_id_prev1=previous_application_onehot[['SK_ID_CURR','SK_ID_PREV']].groupby('SK_ID_CURR' ).size()<groupby>
top_prod_countries = [m[0] for m in Counter(i for j in List_of_countries for i in j ).most_common(6)]
for t in top_prod_countries:
train['prod_country_' + t] = train['all_prod_countries'].apply(lambda x: 1 if t in x else 0)
 test['prod_country_'+ t] = test['all_prod_countries'].apply(lambda x: 1 if t in x else 0 )
previous_application_min=previous_application_onehot.groupby('SK_ID_CURR' ).min().drop('SK_ID_PREV',axis=1)
 previous_application_max=previous_application_onehot.groupby('SK_ID_CURR' ).max().drop('SK_ID_PREV',axis=1)
 previous_application_median=previous_application_onehot.groupby('SK_ID_CURR' ).median().drop('SK_ID_PREV',axis=1)
<merge>
for i, e in enumerate(train['spoken_languages'][:2]):
print(i,e )
previous_application_merge=previous_application_mean.merge(previous_application_min,on='SK_ID_CURR' ).merge(previous_application_max,on='SK_ID_CURR' ).merge(previous_application_median,on='SK_ID_CURR')
 previous_application_merge['cnt_id_prev1']=cnt_id_prev1
previous_application_merge.fillna(0,inplace=True)
 previous_application_merge.head()<drop_column>
print('Number of languages for a movie:')
 train['spoken_languages'].apply(lambda x: len(x)if x!={} else 0 ).value_counts()
del previous_application,previous_application_max,previous_application_mean,previous_application_min,previous_application_onehot
gc.collect()<count_values>
list_of_langs = list(train['spoken_languages'].apply(lambda x: [i['name'] for i in x]
if x!= {} else []))
top_langs = [m[0] for m in Counter(i for j in list_of_langs for i in j ).most_common(5)]
Counter(i for j in list_of_langs for i in j ).most_common(5 )
POS_CASH_balance.NAME_CONTRACT_STATUS.value_counts()<count_missing_values>
train['num_of_langs'] = train['spoken_languages'].apply(lambda x: len(x)if x!= {} else 0)
 test['num_of_langs'] = test['spoken_languages'].apply(lambda x: len(x)if x!= {} else 0)
 train['all_langs'] = train['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name']for i in x]))
if x!= {} else '')
 test['all_langs'] = test['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x]))
if x!= {} else '')
 for l in top_langs:
train['lang_' + l] = train['all_langs'].apply(lambda x: 1 if l in x else 0)
 test['lang_'+ l] = test['all_langs'].apply(lambda x: 1 if l in x else 0 )
check_missing_data(POS_CASH_balance )<data_type_conversions>
for i, e in enumerate(train['Keywords'][:2]):
print(i,e )
POS_CASH_balance.CNT_INSTALMENT_FUTURE.fillna(POS_CASH_balance.CNT_INSTALMENT_FUTURE.median() ,inplace=True )<drop_column>
list_of_keys = list(train['Keywords'].apply(lambda x: [i['name'] for i in x] if x!= {} else []))
Counter(i for j in list_of_keys for i in j ).most_common(10 )
POS_CASH_balance.drop('CNT_INSTALMENT',axis=1,inplace=True )<categorify>
top_keywords = [m[0] for m in Counter(i for j in list_of_keys for i in j ).most_common(10)]
train['num_of_keywords'] = train['Keywords'].apply(lambda x: len(x)if x!={} else 0)
 test['num_of_keywords'] = test['Keywords'].apply(lambda x: len(x)if x!={} else 0)
 train['all_keywords'] = train['Keywords'].apply(lambda x: ' '.join(sorted([i['name']for i in x]))
if x!= {} else '')
 test['all_keywords'] = test['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x]))
if x!={} else '')
 for k in top_keywords:
train['keyword_'+ k] = train['all_keywords'].apply(lambda x: 1 if k in x else 0)
 test['keyword_'+ k] = test['all_keywords'].apply(lambda x: 1 if k in x else 0)
POS_CASH_balance_onehot=onehot_encoding(POS_CASH_balance,categorical_features(POS_CASH_balance))
POS_CASH_balance_onehot.head()<groupby>
for i, e in enumerate(train['cast'][:1]):
print(i,e )
cnt_id_prev2=POS_CASH_balance_onehot[['SK_ID_CURR','SK_ID_PREV']].groupby('SK_ID_CURR' ).size()<drop_column>
print('Number of casts used per movie:')
 train['cast'].apply(lambda x: len(x)if x!={} else 0 ).value_counts().head(10 )
del POS_CASH_balance,POS_CASH_balance_onehot
gc.collect()<feature_engineering>
list_cast_name = list(train['cast'].apply(lambda x: [i['name'] for i in x]if x!= {} else []))
top_cast_name = [m[0] for m in Counter(i for j in list_cast_name for i in j ).most_common(20)]
Counter(i for j in list_cast_name for i in j ).most_common(20 )
categorical_features(credit_card_balance )<count_values>
train['num_of_cast']= train['cast'].apply(lambda x: len(x)if x!={} else 0)
 test['num_of_cast'] = test['cast'].apply(lambda x: len(x)if x!={} else 0)
 train['all_cast_name'] = train['cast'].apply(lambda x: ' '.join(sorted([i['name']for i in x]))
if x!={} else '')
 test['all_cast_name'] = test['cast'].apply(lambda x: ' '.join(sorted([i['name']for i in x]))
if x!= {} else '')
 for c in top_cast_name:
train['cast_name_'+ c]= train['all_cast_name'].apply(lambda x: 1 if c in x else 0)
 test['cast_name_'+ c]= test['all_cast_name'].apply(lambda x: 1 if c in x else 0 )
credit_card_balance.NAME_CONTRACT_STATUS.value_counts()<categorify>
print('Number of crew members per movie:')
 train['crew'].apply(lambda x: len(x)if x!= {} else 0 ).value_counts().head(10 )
credit_card_balance_onehot=onehot_encoding(credit_card_balance,categorical_features(credit_card_balance))<data_type_conversions>
list_crew_names = list(train['crew'].apply(lambda x: [i['name'] for i in x] if x!= {} else [] ).values)
 Counter(i for j in list_crew_names for i in j ).most_common(15 )
credit_card_balance_onehot.fillna(credit_card_balance_onehot.median() ,inplace=True)
 credit_card_balance.head()<groupby>
top_crew_names = [m[0] for m in Counter(i for j in list_crew_names for i in j ).most_common(20)]
train['num_of_crew'] = train['crew'].apply(lambda x: len(x)if x!= {} else 0)
 test['num_of_crew']= test['crew'].apply(lambda x: len(x)if x!= {} else 0)
 for cn in top_crew_names:
train['crew_name_'+ cn]= train['crew'].apply(lambda x: 1 if cn in str(x)else 0)
 test['crew_name_'+ cn] = test['crew'].apply(lambda x: 1 if cn in str(x)else 0 )
cnt_id_prev3=credit_card_balance_onehot[['SK_ID_CURR','SK_ID_PREV']].groupby('SK_ID_CURR' ).size()<drop_column>
train['homepage'].isna().sum()
del credit_card_balance,credit_card_balance_onehot
gc.collect()<count_missing_values>
train['has_homepage'] = 1
train.loc[pd.isnull(train['homepage']),"has_homepage"] = 0
test['has_homepage'] = 1
test.loc[pd.isnull(test['homepage']),"has_homepage"] = 0
check_missing_data(installments_payments )<feature_engineering>
train['runtime'].isna().sum()
categorical_features(installments_payments )<correct_missing_values>
train['runtime'].fillna(train['runtime'].mean() ,inplace= True )
installments_payments.dropna(inplace=True )<groupby>
test['runtime'].fillna(test['runtime'].mean() ,inplace= True )
cnt_id_prev4=installments_payments[['SK_ID_CURR','SK_ID_PREV']].groupby('SK_ID_CURR' ).size()<groupby>
train.loc[train['release_date'].isnull() == True, 'release_date'] = '01/01/98'
test.loc[test['release_date'].isnull() == True, 'release_date'] = '01/01/98'
installments_payments_min=installments_payments.groupby('SK_ID_CURR' ).min().drop('SK_ID_PREV',axis=1)
 installments_payments_max=installments_payments.groupby('SK_ID_CURR' ).max().drop('SK_ID_PREV',axis=1)
 installments_payments_median=installments_payments.groupby('SK_ID_CURR' ).median().drop('SK_ID_PREV',axis=1 )<merge>
def fix_date(x):

 year = x.split('/')[2]
if int(year)<= 19:
return x[:-2] + '20' + year
else:
return x[:-2] + '19' + year
installments_payments_merge=installments_payments_min.merge(installments_payments_max,on='SK_ID_CURR' ).merge(installments_payments_median,on='SK_ID_CURR' )<prepare_output>
train['release_date'] = train['release_date'].apply(lambda x: fix_date(x))
test['release_date'] = test['release_date'].apply(lambda x: fix_date(x))
train['release_date'] = pd.to_datetime(train['release_date'])
 test['release_date'] = pd.to_datetime(test['release_date'] )
installments_payments_merge['cnt_id_prev4']=cnt_id_prev4
installments_payments_merge.fillna(0,inplace=True)
 installments_payments_merge.head()<drop_column>
def process_date(df):
date_parts = ["year", "weekday", "month", 'weekofyear', 'day', 'quarter']
for part in date_parts:
part_col = 'release' + "_" + part
df[part_col] = getattr(df['release_date'].dt, part ).astype(int)
 return df
train = process_date(train)
 test = process_date(test )
del installments_payments,installments_payments_max,installments_payments_min
gc.collect()<define_variables>
d = train['release_date'].dt.year.value_counts().sort_index()
g = train.groupby('release_date')['revenue'].sum()
target=application_train['TARGET']<drop_column>
train['status'].value_counts()
application_train.drop('TARGET',axis=1,inplace=True )<feature_engineering>
rating_na = train.groupby(["release_year","original_language"])['rating'].mean().reset_index()
train[train.rating.isna() ]['rating'] = train.merge(rating_na, how = 'left' ,on = ["release_year","original_language"])
 vote_count_na = train.groupby(["release_year","original_language"])['totalVotes'].mean().reset_index()
train[train.totalVotes.isna() ]['totalVotes'] = train.merge(vote_count_na, how = 'left' ,on = ["release_year","original_language"])
 train['weightedRating'] =(train['rating']*train['totalVotes'] + 6.367 * 1000)/(train['totalVotes'] + 1000)
 train['inflationBudget'] = np.log1p(np.expm1(train['budget'])+ np.expm1(train['budget'])*1.8/100*(2018-train['release_year']))
train['_popularity_mean_year'] = train['popularity'] / train.groupby("release_year")["popularity"].transform('mean')
 train['_budget_runtime_ratio'] = train['budget']/train['runtime']
train['_budget_popularity_ratio'] = train['budget']/train['popularity']
train['_budget_year_ratio'] = train['budget']/(train['release_year']*train['release_year'])
 train['_releaseYear_popularity_ratio'] = train['release_year']/train['popularity']
train['_popularity_totalVotes_ratio'] = train['totalVotes']/train['popularity']
train['_rating_popularity_ratio'] = train['rating']/train['popularity']
train['_rating_totalVotes_ratio'] = train['totalVotes']/train['rating']
train['_totalVotes_releaseYear_ratio'] = train['totalVotes']/train['release_year']
train['_budget_rating_ratio'] = train['budget']/train['rating']
train['_runtime_rating_ratio'] = train['runtime']/train['rating']
train['_budget_totalVotes_ratio'] = train['budget']/train['totalVotes']
train['meanruntimeByYear'] = train.groupby("release_year")["runtime"].aggregate('mean')
 train['meanPopularityByYear'] = train.groupby("release_year")["popularity"].aggregate('mean')
 train['meanBudgetByYear'] = train.groupby("release_year")["budget"].aggregate('mean')
 train['meantotalVotesByYear'] = train.groupby("release_year")["totalVotes"].aggregate('mean')
 train['meanTotalVotesByRating'] = train.groupby("rating")["totalVotes"].aggregate('mean')
 train['isTaglineNA'] = 0
train.loc[train['tagline'] == 0 ,"isTaglineNA"] = 1
train['isTitleDifferent'] = 1
train.loc[ train['original_title'] == train['title'] ,"isTitleDifferent"] = 0
application_train['TARGET']=target
application_train.head()<feature_engineering>
rating_na = test.groupby(["release_year","original_language"])['rating'].mean().reset_index()
test[test.rating.isna() ]['rating'] = test.merge(rating_na, how = 'left' ,on = ["release_year","original_language"])
 vote_count_na = test.groupby(["release_year","original_language"])['totalVotes'].mean().reset_index()
test[test.totalVotes.isna() ]['totalVotes'] = test.merge(vote_count_na, how = 'left' ,on = ["release_year","original_language"])
 test['weightedRating'] =(test['rating']*test['totalVotes'] + 6.367 * 1000)/(test['totalVotes'] + 1000)
 test['inflationBudget'] = np.log1p(np.expm1(test['budget'])+ np.expm1(test['budget'])*1.8/100*(2018-test['release_year']))
test['_popularity_mean_year'] = test['popularity'] / test.groupby("release_year")["popularity"].transform('mean')
 test['_budget_runtime_ratio'] = test['budget']/test['runtime']
test['_budget_popularity_ratio'] = test['budget']/test['popularity']
test['_budget_year_ratio'] = test['budget']/(test['release_year']*test['release_year'])
 test['_releaseYear_popularity_ratio'] = test['release_year']/train['popularity']
test['_popularity_totalVotes_ratio'] = test['totalVotes']/test['popularity']
test['_rating_popularity_ratio'] = test['rating']/test['popularity']
test['_rating_totalVotes_ratio'] = test['totalVotes']/test['rating']
test['_totalVotes_releaseYear_ratio'] = test['totalVotes']/test['release_year']
test['_budget_rating_ratio'] = test['budget']/test['rating']
test['_runtime_rating_ratio'] = test['runtime']/test['rating']
test['_budget_totalVotes_ratio'] = test['budget']/test['totalVotes']
test['meanruntimeByYear'] = test.groupby("release_year")["runtime"].aggregate('mean')
 test['meanPopularityByYear'] = test.groupby("release_year")["popularity"].aggregate('mean')
 test['meanBudgetByYear'] = test.groupby("release_year")["budget"].aggregate('mean')
 test['meantotalVotesByYear'] = test.groupby("release_year")["totalVotes"].aggregate('mean')
 test['meanTotalVotesByRating'] = test.groupby("rating")["totalVotes"].aggregate('mean')
 test['isTaglineNA'] = 0
test.loc[test['tagline'] == 0 ,"isTaglineNA"] = 1
test['isTitleDifferent'] = 1
test.loc[ test['original_title'] == test['title'] ,"isTitleDifferent"] = 0
application_test['TARGET']=-999<concatenate>
train = train.drop(['id','belongs_to_collection','genres','homepage','imdb_id','overview','runtime'
,'poster_path','production_companies','production_countries','release_date','spoken_languages'
,'status','title','Keywords','cast','crew','original_language','original_title','tagline','all_genres',
'all_prod_companies','all_prod_countries','all_langs','all_keywords','all_cast_name'],axis=1)
 test = test.drop(['id','belongs_to_collection','genres','homepage','imdb_id','overview','runtime'
,'poster_path','production_companies','production_countries','release_date','spoken_languages'
,'status','title','Keywords','cast','crew','original_language','original_title','tagline','all_genres',
'all_prod_companies','all_prod_countries','all_langs','all_keywords','all_cast_name'],axis=1 )
df=pd.concat([application_train,application_test] )<feature_engineering>
train.fillna(value=0.0, inplace = True)
 test.fillna(value=0.0, inplace = True )
categorical_features(df )<categorify>
def clean_dataset(df):
assert isinstance(df, pd.DataFrame), "df needs to be a pd.DataFrame"
df.dropna(inplace=True)
 indices_to_keep = ~df.isin([np.nan, np.inf, -np.inf] ).any(1)
 return df[indices_to_keep].astype(np.float64)
 train = clean_dataset(train )
df_onehot=onehot_encoding(df,categorical_features(df))
df_onehot.shape<correct_missing_values>
X = train.drop(['revenue'],axis=1)
 y = train.revenue
X_train, X_valid, y_train, y_valid = train_test_split(X,y,test_size=0.2,random_state=25 )
df_onehot.fillna(0,inplace=True )<drop_column>
lr = LinearRegression()
lr.fit(X_train, y_train)
 pred = lr.predict(X_valid)
 accuracy = r2_score(y_valid,pred)
 print('Linear Regression R2 Score: ', accuracy)
 mse = mean_squared_error(y_valid,pred)
 print('Mean Squared Error: ', mse)
 print('Root Mean Square Error',np.sqrt(mse))
cv_pred = cross_val_predict(lr,X,y,n_jobs=-1, cv=10)
 cv_accuracy = r2_score(y,cv_pred)
 print('Cross-Predicted(KFold)R2 Score: ', cv_accuracy)
del application_test,application_train,df
gc.collect()<merge>
ls = Lasso()
ls.fit(X_train, y_train)
 pred = ls.predict(X_valid)
 accuracy = r2_score(y_valid,pred)
 print('Lasso Regression R2 Score: ', accuracy)
 mse = mean_squared_error(y_valid,pred)
 print('Mean Squared Error: ', mse)
 print('Root Mean Squared Error', np.sqrt(mse))
cv_pred = cross_val_predict(ls,X,y,n_jobs=-1, cv=10)
 cv_accuracy = r2_score(y,cv_pred)
 print('Cross-Predicted(KFold)Lasso Regression Accuracy: ', cv_accuracy )
total=df_onehot.merge(right=bureau_final_median,on='SK_ID_CURR',how='left' ).merge(right=previous_application_median,on='SK_ID_CURR',how='left' ).merge(right=POS_CASH_balance_median,on='SK_ID_CURR',how='left' ).merge(right=credit_card_balance_median,on='SK_ID_CURR',how='left' ).merge(right=installments_payments_merge,on='SK_ID_CURR',how='left')
 total.shape<drop_column>
dt = DecisionTreeRegressor()
dt.fit(X_train, y_train)
 pred = dt.predict(X_valid)
 accuracy = r2_score(y_valid,pred)
 print('Decision Tree R2 Score: ', accuracy)
 mse = mean_squared_error(y_valid,pred)
 print('Mean Squared Error: ', mse)
 print('Root Mean Square Error',np.sqrt(mse))
cv_pred = cross_val_predict(dt,X,y,n_jobs=-1, cv=10)
 cv_accuracy = r2_score(y,cv_pred)
 print('Cross-Predicted(KFold)Decision Tree Accuracy: ', cv_accuracy )
del total,df_onehot,bureau_final_median,previous_application_merge,previous_application_median
del POS_CASH_balance_median,credit_card_balance_median,installments_payments_median,installments_payments_merge
gc.collect()<filter>
rf = RandomForestRegressor()
rf.fit(X_train, y_train)
 pred = rf.predict(X_valid)
 accuracy = r2_score(y_valid,pred)
 print('Random Forest Regressor R2: ', accuracy)
 mse = mean_squared_error(y_valid,pred)
 print('Mean Squared Error: ', mse)
 print('Root Mean Square Error',np.sqrt(mse))
cv_pred = cross_val_predict(rf,X,y,n_jobs=-1, cv=10)
 cv_accuracy = r2_score(y,cv_pred)
 print('Cross-Predicted(KFold)Random Forest R2: ', cv_accuracy )
df_train=df_total[df_total.TARGET!=-999] 
<filter>
rfr = RandomForestRegressor()
n_estimators = [int(x)for x in np.linspace(start = 50 , stop = 300, num = 5)]
max_features = [10,20,40,60,80,100,120]
max_depth = [int(x)for x in np.linspace(5, 10, num = 2)]
max_depth.append(None)
 bootstrap = [True, False]
r_grid = {'n_estimators': n_estimators,
'max_features': max_features,
'max_depth': max_depth,
'bootstrap': bootstrap}
cv_random = RandomizedSearchCV(estimator=rfr, param_distributions=r_grid, n_iter = 20,
scoring='neg_mean_squared_error', cv = 3, verbose=2, random_state=42,
n_jobs=-1, return_train_score=True)
 cv_random.fit(X_train, y_train);
print(cv_random.best_params_)
 pred = cv_random.predict(X_valid)
 mse = mean_squared_error(y_valid,pred)
 print('Mean Squared Error: ', mse)
 print('Root Mean Square Error',np.sqrt(mse))
cv_accuracy = r2_score(y_valid,pred)
 print('Random Forest Predict R2: ', cv_accuracy )
df_test=df_total[df_total.TARGET==-999] 
<drop_column>
import h2o
from h2o.estimators.gbm import H2OGradientBoostingEstimator
from h2o.automl import H2OAutoML
test=df_test.drop(columns=["SK_ID_CURR",'TARGET'],axis=1)
 test.shape<prepare_x_and_y>
h2o.init()
y=df_train['TARGET'].values
y<drop_column>
h2o_df=h2o.H2OFrame(train)
 h2o_df.head()
train=df_train.drop(columns=["SK_ID_CURR",'TARGET'],axis=1 ).values
train.shape<drop_column>
splits = h2o_df.split_frame(ratios=[0.8],seed=1)
 h2o_train = splits[0]
h2o_valid = splits[1]
del df_train,df_test,df_total
gc.collect()<set_options>
y = "revenue"
x = h2o_df.columns
x.remove(y )
gc.collect()<import_modules>
aml = H2OAutoML(max_runtime_secs=180, seed=1,stopping_metric='RMSE' )
from sklearn.model_selection import train_test_split<import_modules>
aml.train(x=x,y=y, training_frame=h2o_train )
from sklearn.model_selection import train_test_split<split>
pred = aml.predict(h2o_valid)
 pred.head()
X_train,X_test,y_train,y_test=train_test_split(train,y,test_size=0.2 )<set_options>
h2o.save_model(aml.leader, path="./model_bin" )
del train
gc.collect()<import_modules>
params = {'objective': 'reg:linear',
'eta': 0.01,
'max_depth': 6,
'min_child_weight': 3,
'subsample': 0.8,
'colsample_bytree': 0.8,
'colsample_bylevel': 0.50,
'gamma': 1.45,
'eval_metric': 'rmse',
'seed': 12,
'silent': True}
xgb_data = [(xgb.DMatrix(X_train, y_train), 'train'),(xgb.DMatrix(X_valid, y_valid), 'valid')]
print('Starting training...')
 xgb_model = xgb.train(params,
xgb.DMatrix(X_train, y_train),
10000,
xgb_data,
verbose_eval=300,
early_stopping_rounds=300 )
import lightgbm<create_dataframe>
xgb_pred = xgb_model.predict(xgb.DMatrix(X_valid))
train_data=lightgbm.Dataset(X_train,label=y_train)
 valid_data=lightgbm.Dataset(X_test,label=y_test )<init_hyperparams>
X_test = test.drop('revenue',axis=1 )
params = {'boosting_type': 'gbdt',
'max_depth' : 10,
'objective': 'binary',
'nthread': 5,
'num_leaves': 64,
'learning_rate': 0.1,
'max_bin': 512,
'subsample_for_bin': 200,
'subsample': 1,
'subsample_freq': 1,
'colsample_bytree': 0.8,
'reg_alpha': 5,
'reg_lambda': 10,
'min_split_gain': 0.005,
'min_child_weight': 1,
'min_child_samples': 5,
'scale_pos_weight': 1,
'num_class' : 1,
'metric' : 'auc'
}<train_model>
X_test[X_test==np.inf]=np.nan
X_test.fillna(X_test.mean() , inplace=True )
lgbm = lightgbm.train(params,
train_data,
25000,
valid_sets=valid_data,
early_stopping_rounds= 80,
verbose_eval= 10
)<predict_on_test>
test_pred_xgb = xgb_model.predict(xgb.DMatrix(( X_test)) , ntree_limit=xgb_model.best_ntree_limit )
predictions_lgbm_prob = lgbm.predict(test.values )<load_from_csv>
model = CatBoostRegressor(iterations=100000,
learning_rate=0.005,
depth=5,
eval_metric='RMSE',
colsample_bylevel=0.8,
random_seed = 21,
bagging_temperature = 0.2,
metric_period = None,
early_stopping_rounds=200 
)
 model.fit(X_train, y_train,eval_set=(X_valid, y_valid),use_best_model=True,verbose=500)
 val_pred = model.predict(X_valid)
 print('RMSE',np.sqrt(mean_squared_error(val_pred,y_valid)))
 test_pred_cat = model.predict(X_test )
sub=pd.read_csv('.. /input/sample_submission.csv' )<prepare_output>
params = {'objective':'regression',
'num_leaves' : 30,
'min_data_in_leaf' : 20,
'max_depth' : 9,
'learning_rate': 0.004,
'feature_fraction':0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
'lambda_l1': 0.2,
"bagging_seed": 11,
"metric": 'rmse',
"random_state" : 11,
"verbosity": -1}
record = dict()
model = lgb.train(params
, lgb.Dataset(X_train, y_train)
 , num_boost_round = 100000
, valid_sets = [lgb.Dataset(X_valid, y_valid)]
, verbose_eval = 500
, early_stopping_rounds = 500
, callbacks = [lgb.record_evaluation(record)] 
)
 best_idx = np.argmin(np.array(record['valid_0']['rmse']))
val_pred = model.predict(X_valid, num_iteration = model.best_iteration)
 test_pred_gbm = model.predict(X_test, num_iteration = model.best_iteration )
<save_to_csv><EOS> 
sub = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 df_sub = pd.DataFrame()
df_sub['id'] = sub['id']
final_pred = 0.3*test_pred_xgb + 0.7*test_pred_cat
df_sub['revenue'] = np.expm1(final_pred)
 print(df_sub['revenue'])
 df_sub.to_csv("submission.csv", index=False )