Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<merge>
pd.set_option('max_columns', None)
 %matplotlib inline
plt.style.use('ggplot')
 stop = set(stopwords.words('english'))
warnings.filterwarnings("ignore" )
install_AMT_DIFF_mean_mean = install_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['install_AMT_DIFF_mean'].mean().rename(columns = {'install_AMT_DIFF_mean': 'install_AMT_DIFF_mean_mean'})
 dataset = dataset.merge(install_AMT_DIFF_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del install_AMT_DIFF_mean_mean
gc.collect()<merge>
train=pd.read_csv('.. /input/train.csv',index_col=0)
 test=pd.read_csv('.. /input/test.csv',index_col=0 )
install_AMT_DIFF_max_mean = install_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['install_AMT_DIFF_max'].mean().rename(columns = {'install_AMT_DIFF_max': 'install_AMT_DIFF_max_mean'})
 dataset = dataset.merge(install_AMT_DIFF_max_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del install_AMT_DIFF_max_mean
gc.collect()<merge>
test['revenue']=-99
install_AMT_DIFF_min_mean = install_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['install_AMT_DIFF_min'].mean().rename(columns = {'install_AMT_DIFF_min': 'install_AMT_DIFF_min_mean'})
 dataset = dataset.merge(install_AMT_DIFF_min_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del install_AMT_DIFF_min_mean
gc.collect()<set_options>
train=train.drop('belongs_to_collection',axis=1)
 test=test.drop('belongs_to_collection',axis=1 )
gc.enable()
del install, install_stats_by_prev
gc.collect()<load_from_csv>
train.loc[train.index == 16,'revenue'] = 192864
train.loc[train.index == 90,'budget'] = 30000000
train.loc[train.index== 118,'budget'] = 60000000
train.loc[train.index== 149,'budget'] = 18000000
train.loc[train.index== 313,'revenue'] = 12000000
train.loc[train.index == 451,'revenue'] = 12000000
train.loc[train.index == 464,'budget'] = 20000000
train.loc[train.index == 470,'budget'] = 13000000
train.loc[train.index== 513,'budget'] = 930000
train.loc[train.index == 797,'budget'] = 8000000
train.loc[train.index == 819,'budget'] = 90000000
train.loc[train.index == 850,'budget'] = 90000000
train.loc[train.index == 1007,'budget'] = 2
train.loc[train.index== 1112,'budget'] = 7500000
train.loc[train.index == 1131,'budget'] = 4300000
train.loc[train.index == 1359,'budget'] = 10000000
train.loc[train.index == 1542,'budget'] = 1
train.loc[train.index == 1570,'budget'] = 15800000
train.loc[train.index== 1571,'budget'] = 4000000
train.loc[train.index == 1714,'budget'] = 46000000
train.loc[train.index == 1721,'budget'] = 17500000
train.loc[train.index== 1865,'revenue'] = 25000000
train.loc[train.index == 1885,'budget'] = 12
train.loc[train.index == 2091,'budget'] = 10
train.loc[train.index == 2268,'budget'] = 17500000
train.loc[train.index == 2491,'budget'] = 6
train.loc[train.index == 2602,'budget'] = 31000000
train.loc[train.index == 2612,'budget'] = 15000000
train.loc[train.index == 2696,'budget'] = 10000000
train.loc[train.index == 2801,'budget'] = 10000000
train.loc[train.index == 335,'budget'] = 2
train.loc[train.index == 348,'budget'] = 12
train.loc[train.index == 470,'budget'] = 13000000
train.loc[train.index == 513,'budget'] = 1100000
train.loc[train.index == 640,'budget'] = 6
train.loc[train.index == 696,'budget'] = 1
train.loc[train.index == 797,'budget'] = 8000000
train.loc[train.index == 850,'budget'] = 1500000
train.loc[train.index == 1199,'budget'] = 5
train.loc[train.index == 1282,'budget'] = 9
train.loc[train.index== 1347,'budget'] = 1
train.loc[train.index== 1755,'budget'] = 2
train.loc[train.index == 1801,'budget'] = 5
train.loc[train.index == 1918,'budget'] = 592
train.loc[train.index == 2033,'budget'] = 4
train.loc[train.index == 2118,'budget'] = 344
train.loc[train.index == 2252,'budget'] = 130
train.loc[train.index == 2256,'budget'] = 1
train.loc[train.index == 2696,'budget'] = 10000000
cash = pd.read_csv('.. /input/POS_CASH_balance.csv')
 cash.head()<define_variables>
test.loc[test.index== 3033,'budget'] = 250
test.loc[test.index== 3051,'budget'] = 50
test.loc[test.index == 3084,'budget'] = 337
test.loc[test.index == 3224,'budget'] = 4
test.loc[test.index == 3594,'budget'] = 25
test.loc[test.index == 3619,'budget'] = 500
test.loc[test.index == 3831,'budget'] = 3
test.loc[test.index== 3935,'budget'] = 500
test.loc[test.index == 4049,'budget'] = 995946
test.loc[test.index== 4424,'budget'] = 3
test.loc[test.index == 4460,'budget'] = 8
test.loc[test.index == 4555,'budget'] = 1200000
test.loc[test.index== 4624,'budget'] = 30
test.loc[test.index== 4645,'budget'] = 500
test.loc[test.index == 4709,'budget'] = 450
test.loc[test.index == 4839,'budget'] = 7
test.loc[test.index== 3125,'budget'] = 25
test.loc[test.index== 3142,'budget'] = 1
test.loc[test.index == 3201,'budget'] = 450
test.loc[test.index == 3222,'budget'] = 6
test.loc[test.index== 3545,'budget'] = 38
test.loc[test.index == 3670,'budget'] = 18
test.loc[test.index == 3792,'budget'] = 19
test.loc[test.index == 3881,'budget'] = 7
test.loc[test.index == 3969,'budget'] = 400
test.loc[test.index == 4196,'budget'] = 6
test.loc[test.index == 4221,'budget'] = 11
test.loc[test.index == 4222,'budget'] = 500
test.loc[test.index== 4285,'budget'] = 11
test.loc[test.index == 4319,'budget'] = 1
test.loc[test.index == 4639,'budget'] = 10
test.loc[test.index == 4719,'budget'] = 45
test.loc[test.index == 4822,'budget'] = 22
test.loc[test.index == 4829,'budget'] = 20
test.loc[test.index== 4969,'budget'] = 20
test.loc[test.index== 5021,'budget'] = 40
test.loc[test.index== 5035,'budget'] = 1
test.loc[test.index== 5063,'budget'] = 14
test.loc[test.index == 5119,'budget'] = 2
test.loc[test.index== 5214,'budget'] = 30
test.loc[test.index== 5221,'budget'] = 50
test.loc[test.index== 4903,'budget'] = 15
test.loc[test.index == 4983,'budget'] = 3
test.loc[test.index == 5102,'budget'] = 28
test.loc[test.index== 5217,'budget'] = 75
test.loc[test.index == 5224,'budget'] = 3
test.loc[test.index== 5469,'budget'] = 20
test.loc[test.index == 5840,'budget'] = 1
test.loc[test.index == 5960,'budget'] = 30
test.loc[test.index == 6506,'budget'] = 11
test.loc[test.index== 6553,'budget'] = 280
test.loc[test.index == 6561,'budget'] = 7
test.loc[test.index== 6582,'budget'] = 218
test.loc[test.index == 6638,'budget'] = 5
test.loc[test.index== 6749,'budget'] = 8
test.loc[test.index==6759,'budget'] = 50
test.loc[test.index == 6856,'budget'] = 10
test.loc[test.index== 6858,'budget'] = 100
test.loc[test.index == 6876,'budget'] = 250
test.loc[test.index == 6972,'budget'] = 1
test.loc[test.index== 7079,'budget'] = 8000000
test.loc[test.index == 7150,'budget'] = 118
test.loc[test.index == 6506,'budget'] = 118
test.loc[test.index == 7225,'budget'] = 6
test.loc[test.index == 7231,'budget'] = 85
test.loc[test.index == 5222,'budget'] = 5
test.loc[test.index == 5322,'budget'] = 90
test.loc[test.index == 5350,'budget'] = 70
test.loc[test.index == 5378,'budget'] = 10
test.loc[test.index== 5545,'budget'] = 80
test.loc[test.index == 5810,'budget'] = 8
test.loc[test.index== 5926,'budget'] = 300
test.loc[test.index== 5927,'budget'] = 4
test.loc[test.index== 5986,'budget'] = 1
test.loc[test.index == 6053,'budget'] = 20
test.loc[test.index== 6104,'budget'] = 1
test.loc[test.index == 6130,'budget'] = 30
test.loc[test.index == 6301,'budget'] = 150
test.loc[test.index == 6276,'budget'] = 100
test.loc[test.index == 6473,'budget'] = 100
test.loc[test.index== 6842,'budget'] = 30
cash_stats_by_prev = cash[['SK_ID_PREV', 'SK_ID_CURR']]<merge>
new_data=pd.concat([train,test],axis=0)
 new_data.head()
cash_MONTHS_BALANCE_count = cash.groupby('SK_ID_PREV', as_index=False)['MONTHS_BALANCE'].count().rename(columns = {'MONTHS_BALANCE': 'cash_MONTHS_BALANCE_count'})
 cash_MONTHS_BALANCE_mean = cash.groupby('SK_ID_PREV', as_index=False)['MONTHS_BALANCE'].mean().rename(columns = {'MONTHS_BALANCE': 'cash_MONTHS_BALANCE_mean'})
 cash_stats_by_prev = cash_stats_by_prev.merge(cash_MONTHS_BALANCE_count, on = 'SK_ID_PREV', how = 'left')
 cash_stats_by_prev = cash_stats_by_prev.merge(cash_MONTHS_BALANCE_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del cash_MONTHS_BALANCE_count, cash_MONTHS_BALANCE_mean
gc.collect()<merge>
new_data.isnull().sum()
cash_CNT_INSTALMENT_mean = cash.groupby('SK_ID_PREV', as_index=False)['CNT_INSTALMENT'].mean().rename(columns = {'CNT_INSTALMENT': 'cash_CNT_INSTALMENT_mean'})
 cash_stats_by_prev = cash_stats_by_prev.merge(cash_CNT_INSTALMENT_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del cash_CNT_INSTALMENT_mean
gc.collect()<merge>
new_data[new_data['release_date'].isnull() ]
cash_CNT_INSTALMENT_FUTURE_mean = cash.groupby('SK_ID_PREV', as_index=False)['CNT_INSTALMENT_FUTURE'].mean().rename(columns = {'CNT_INSTALMENT_FUTURE': 'cash_CNT_INSTALMENT_FUTURE_mean'})
 cash_stats_by_prev = cash_stats_by_prev.merge(cash_CNT_INSTALMENT_FUTURE_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del cash_CNT_INSTALMENT_FUTURE_mean
gc.collect()<merge>
new_data['release_date']=new_data['release_date'].fillna('3/20/01' )
cash_SK_DPD_mean = cash.groupby('SK_ID_PREV', as_index=False)['SK_DPD'].mean().rename(columns = {'SK_DPD': 'cash_SK_DPD_mean'})
 cash_stats_by_prev = cash_stats_by_prev.merge(cash_SK_DPD_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del cash_SK_DPD_mean
gc.collect()<merge>
new_data['release_year']=pd.to_datetime(new_data['release_date'] ).dt.year
new_data['release_month']=pd.to_datetime(new_data['release_date'] ).dt.month
new_data['release_day']=pd.to_datetime(new_data['release_date'] ).dt.day
cash_SK_DPD_DEF_mean = cash.groupby('SK_ID_PREV', as_index=False)['SK_DPD_DEF'].mean().rename(columns = {'SK_DPD_DEF': 'cash_SK_DPD_DEF_mean'})
 cash_stats_by_prev = cash_stats_by_prev.merge(cash_SK_DPD_DEF_mean, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del cash_SK_DPD_DEF_mean
gc.collect()<categorify>
new_data['release_year'].loc[new_data['release_year']>=2018]-=100
cash_cats = pd.get_dummies(cash.select_dtypes('object'))
cash_cats['SK_ID_PREV'] = cash['SK_ID_PREV']
cash_cats.head()<groupby>
new_data=new_data.drop('release_date',axis=1 )
cash_cats_grouped = cash_cats.groupby('SK_ID_PREV' ).agg('sum')
 cash_cats_grouped.head()<merge>
new_data['homepage_fact']=new_data['homepage'].apply(lambda x: 0 if x is np.nan else 1 )
cash_stats_by_prev = cash_stats_by_prev.merge(cash_cats_grouped, on = 'SK_ID_PREV', how = 'left')
 gc.enable()
del cash_cats_grouped, cash_cats
gc.collect()<merge>
new_data['homepage_end']=new_data[new_data['homepage'].notna() ]['homepage'].str.findall(r'\.( [a-z]+ )(?:\/|$)' ).apply(lambda x:x[0])
 new_data['homepage_end'].head()
cash_MONTHS_BALANCE_count_mean = cash_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['cash_MONTHS_BALANCE_count'].mean().rename(columns = {'cash_MONTHS_BALANCE_count': 'cash_MONTHS_BALANCE_count_mean'})
 dataset = dataset.merge(cash_MONTHS_BALANCE_count_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del cash_MONTHS_BALANCE_count_mean
gc.collect()<merge>
new_data['homepage_end']=new_data['homepage_end'].fillna('unknow' )
cash_MONTHS_BALANCE_mean_mean = cash_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['cash_MONTHS_BALANCE_mean'].mean().rename(columns = {'cash_MONTHS_BALANCE_mean': 'cash_MONTHS_BALANCE_mean_mean'})
 dataset = dataset.merge(cash_MONTHS_BALANCE_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del cash_MONTHS_BALANCE_mean_mean
gc.collect()<merge>
page=pd.get_dummies(new_data['homepage_end'])
 page.head()
cash_CNT_INSTALMENT_mean_mean = cash_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['cash_CNT_INSTALMENT_mean'].mean().rename(columns = {'cash_CNT_INSTALMENT_mean': 'cash_CNT_INSTALMENT_mean_mean'})
 dataset = dataset.merge(cash_CNT_INSTALMENT_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del cash_CNT_INSTALMENT_mean_mean
gc.collect()<merge>
new_data=new_data.drop('homepage',axis=1 )
cash_CNT_INSTALMENT_FUTURE_mean_mean = cash_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['cash_CNT_INSTALMENT_FUTURE_mean'].mean().rename(columns = {'cash_CNT_INSTALMENT_FUTURE_mean': 'cash_CNT_INSTALMENT_FUTURE_mean_mean'})
 dataset = dataset.merge(cash_CNT_INSTALMENT_FUTURE_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del cash_CNT_INSTALMENT_FUTURE_mean_mean
gc.collect()<merge>
new_data=new_data.drop('poster_path',axis=1 )
cash_SK_DPD_mean_mean = cash_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['cash_SK_DPD_mean'].mean().rename(columns = {'cash_SK_DPD_mean': 'cash_SK_DPD_mean_mean'})
 dataset = dataset.merge(cash_SK_DPD_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del cash_SK_DPD_mean_mean
gc.collect()<merge>
new_data['len_overview']=new_data['overview'].fillna('NAN' ).apply(lambda x:len(x))
cash_SK_DPD_DEF_mean_mean = cash_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['cash_SK_DPD_DEF_mean'].mean().rename(columns = {'cash_SK_DPD_DEF_mean': 'cash_SK_DPD_DEF_mean_mean'})
 dataset = dataset.merge(cash_SK_DPD_DEF_mean_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del cash_SK_DPD_DEF_mean_mean
gc.collect()<merge>
len_rew_sort=new_data['len_overview'].sort_values(ascending=True)
 len_rew_sort.head()
cash_NAME_CONTRACT_STATUS_Active_mean = cash_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_Active'].mean().rename(columns = {'NAME_CONTRACT_STATUS_Active': 'cash_NAME_CONTRACT_STATUS_Active_mean'})
 dataset = dataset.merge(cash_NAME_CONTRACT_STATUS_Active_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del cash_NAME_CONTRACT_STATUS_Active_mean
gc.collect()<merge>
length=len(new_data['len_overview'])
 m=0.1
n=0.1
arr_len_ove=[]
for i in range(1,11):
arr_len_ove.append(round(length*m))
m+=n
arr_len_ove
cash_NAME_CONTRACT_STATUS_Amortized_mean = cash_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_Amortized debt'].mean().rename(columns = {'NAME_CONTRACT_STATUS_Amortized debt': 'cash_NAME_CONTRACT_STATUS_Amortized_mean'})
 dataset = dataset.merge(cash_NAME_CONTRACT_STATUS_Amortized_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del cash_NAME_CONTRACT_STATUS_Amortized_mean
gc.collect()<merge>
for i in range(10):
qu=qu_arr[i]
if i==0:
new_data['len_overview'].loc[(new_data['len_overview']<len_rew_sort.iloc[arr_len_ove[i]-1])]=qu
else:
new_data['len_overview'].loc[(new_data['len_overview']<len_rew_sort.iloc[arr_len_ove[i]-1])&(new_data['len_overview'] >qu_arr[i-1])]=qu
print(i,qu )
cash_NAME_CONTRACT_STATUS_Approved_mean = cash_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_Approved'].mean().rename(columns = {'NAME_CONTRACT_STATUS_Approved': 'cash_NAME_CONTRACT_STATUS_Approved_mean'})
 dataset = dataset.merge(cash_NAME_CONTRACT_STATUS_Approved_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del cash_NAME_CONTRACT_STATUS_Approved_mean
gc.collect()<merge>
len_ove_agg=new_data.groupby('len_overview' ).revenue.aggregate(['min','max','std'] )
cash_NAME_CONTRACT_STATUS_Canceled_mean = cash_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_Canceled'].mean().rename(columns = {'NAME_CONTRACT_STATUS_Canceled': 'cash_NAME_CONTRACT_STATUS_Canceled_mean'})
 dataset = dataset.merge(cash_NAME_CONTRACT_STATUS_Canceled_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del cash_NAME_CONTRACT_STATUS_Canceled_mean
gc.collect()<merge>
new_data['geres_name']=new_data['genres'].str.findall(r''name'\s?:\s?'(\w+)'')
 new_data['geres_name'].head()
cash_NAME_CONTRACT_STATUS_Completed_mean = cash_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_Completed'].mean().rename(columns = {'NAME_CONTRACT_STATUS_Completed': 'cash_NAME_CONTRACT_STATUS_Completed_mean'})
 dataset = dataset.merge(cash_NAME_CONTRACT_STATUS_Completed_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del cash_NAME_CONTRACT_STATUS_Completed_mean
gc.collect()<merge>
new_data=new_data.drop('genres',axis=1 )
cash_NAME_CONTRACT_STATUS_Demand_mean = cash_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_Demand'].mean().rename(columns = {'NAME_CONTRACT_STATUS_Demand': 'cash_NAME_CONTRACT_STATUS_Demand_mean'})
 dataset = dataset.merge(cash_NAME_CONTRACT_STATUS_Demand_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del cash_NAME_CONTRACT_STATUS_Demand_mean
gc.collect()<merge>
country=new_data['production_countries'].str.findall(r'[A-Z]{2,5}')
cash_NAME_CONTRACT_STATUS_Returned_mean = cash_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_Returned to the store'].mean().rename(columns = {'NAME_CONTRACT_STATUS_Returned to the store': 'cash_NAME_CONTRACT_STATUS_Returned_mean'})
 dataset = dataset.merge(cash_NAME_CONTRACT_STATUS_Returned_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del cash_NAME_CONTRACT_STATUS_Returned_mean
gc.collect()<merge>
new_data['production_countries']=country
cash_NAME_CONTRACT_STATUS_Signed_mean = cash_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_Signed'].mean().rename(columns = {'NAME_CONTRACT_STATUS_Signed': 'cash_NAME_CONTRACT_STATUS_Signed_mean'})
 dataset = dataset.merge(cash_NAME_CONTRACT_STATUS_Signed_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del cash_NAME_CONTRACT_STATUS_Signed_mean
gc.collect()<merge>
new_data['original_and_new']=new_data['original_title']==new_data['title']
cash_NAME_CONTRACT_STATUS_XNA_mean = cash_stats_by_prev.groupby('SK_ID_CURR', as_index=False)['NAME_CONTRACT_STATUS_XNA'].mean().rename(columns = {'NAME_CONTRACT_STATUS_XNA': 'cash_NAME_CONTRACT_STATUS_XNA_mean'})
 dataset = dataset.merge(cash_NAME_CONTRACT_STATUS_XNA_mean, on = 'SK_ID_CURR', how = 'left')
 gc.enable()
del cash_NAME_CONTRACT_STATUS_XNA_mean
gc.collect()<set_options>
new_data=new_data.drop('original_title',axis=1)
 new_data.head()
gc.enable()
del cash, cash_stats_by_prev
gc.collect()<count_values>
new_data=pd.concat([new_data,page],axis=1 )
dataset.dtypes.value_counts()<categorify>
new_data['production_companies']=new_data['production_companies'].str.findall(r''name'?:\s?'([A-Za-z]+)')
 new_data.fillna('Unknow')
 print('接下来就是地图可视化了' )
dataset['bureau_DAYS_CREDIT_ENDDATE_max_outlier'] = dataset['bureau_DAYS_CREDIT_ENDDATE_max_outlier'].map({False:0, True:1})
 dataset['bureau_DAYS_ENDDATE_FACT_mean_outlier'] = dataset['bureau_DAYS_ENDDATE_FACT_mean_outlier'].map({False:0, True:1} )<split>
new_data=new_data.drop('imdb_id',axis=1 )
y_temp = dataset[['TARGET']]
X_temp = dataset.drop(['TARGET'], axis=1)
 X_big, X_small, y_big, y_small = train_test_split(X_temp, y_temp, test_size=0.2, random_state=1 )<data_type_conversions>
new_data['spoken_languages']=new_data['spoken_languages'].str.findall(r''([a-z]{2})'' )
upper_corr = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
upper_corr.head()<define_variables>
new_data['production_companies']=new_data['production_companies'].fillna('unknow' )
drop_cols = [column for column in upper_corr.columns if any(upper_corr[column] > 0.9)]
print('Columns to remove: ', len(drop_cols))<sort_values>
new_data['spoken_languages']=new_data['spoken_languages'].fillna('unknow' )
dataset_missing =(dataset.isnull().sum() / len(dataset)).sort_values(ascending = False)
 dataset_missing.head(10 )<filter>
new_data['Keywords']=new_data['Keywords'].str.findall(r''?:\s?'([a-z]+\s?[a-z]+)'' ).fillna('unkonw' )
dataset_missing = dataset_missing.index[dataset_missing > 0.75]
print('Columns with more than 75% missing values: ', len(dataset_missing))<drop_column>
d = Counter([i for j in list_of_keywords for i in j] ).most_common(30)
 d
train = dataset[:train_len]
x_test = dataset[train_len:]
train_ids = train['SK_ID_CURR']
test_ids = x_test['SK_ID_CURR']
train.drop(columns=['SK_ID_CURR'], axis = 1, inplace=True)
 x_test.drop(columns=['TARGET', 'SK_ID_CURR'], axis = 1, inplace=True )<prepare_x_and_y>
for i in d:
m=i[0]
new_data['Keywords_'+m]=new_data['Keywords'].apply(lambda x:1 if m in x else 0 )
train['TARGET'] = train['TARGET'].astype(int)
 y_train = train['TARGET']
x_train = train.drop(columns=['TARGET'], axis = 1 )<create_dataframe>
new_data=new_data.drop('Keywords',axis=1 )
feature_imp = np.zeros(x_train.shape[1] )<choose_model_class>
new_data['cast']=new_data['cast'].fillna('unknow' )
model = lgb.LGBMClassifier(objective='binary', boosting_type='goss', n_estimators=10000, class_weight='balanced' )<train_model>
for i in d:
m=i[0][0]
new_data['cast_name_'+m]=new_data['cast'].apply(lambda x:1 if m in x else 0 )
for i in range(2):
train_x1, train_x2, train_y1, train_y2 = train_test_split(x_train, y_train, test_size = 0.25, random_state = i)
 model.fit(train_x1, train_y1, early_stopping_rounds=100, eval_set = [(train_x2, train_y2)], eval_metric = 'auc', verbose = 200)
 feature_imp += model.feature_importances_<filter>
list_cast_gender=list(new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?'))
Counter([i for j in list_cast_gender for i in j] ).most_common()
zero_imp = list(feature_imp[feature_imp['importance'] == 0.0]['feature'])
 print('count of features with 0 importance: ', len(zero_imp))
feature_imp.tail(10 )<drop_column>
new_data['cast_gender_0']=new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?' ).apply(lambda x: x.count('0'))
new_data['cast_gender_1']=new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?' ).apply(lambda x: x.count('1'))
new_data['cast_gender_2']=new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?' ).apply(lambda x: x.count('2'))
x_train = x_train.drop(columns = zero_imp)
 x_test = x_test.drop(columns = zero_imp )<define_variables>
for g in top_cast_char:
m=g[0]
new_data['cast_char_'+m]=new_data['cast'].apply(lambda x:1 if m in x else 0 )
test_predictions = np.zeros(x_test.shape[0])
 out_of_fold = np.zeros(x_train.shape[0])
 valid_scores = []
train_scores = []<choose_model_class>
new_data=new_data.drop('cast',axis=1 )
k_fold = KFold(n_splits = 5, shuffle = False, random_state = 50 )<prepare_x_and_y>
new_data['status']=new_data['status'].fillna('Unknow' )
x_train = np.array(x_train)
 x_test = np.array(x_test )<train_model>
new_data['overview']=new_data['overview'].fillna('' )
for train_indices, valid_indices in k_fold.split(x_train):
train_features, train_labels = x_train[train_indices], y_train[train_indices]
valid_features, valid_labels = x_train[valid_indices], y_train[valid_indices]
model = lgb.LGBMClassifier(n_estimators=10000, objective = 'binary', boosting_type='goss',class_weight = 'balanced',
learning_rate = 0.05, reg_alpha = 0.1, reg_lambda = 0.1, n_jobs = -1, random_state = 50)
 model.fit(train_features, train_labels, eval_metric = 'auc',
eval_set = [(valid_features, valid_labels),(train_features, train_labels)],
eval_names = ['valid', 'train'], early_stopping_rounds = 100, verbose = 200)
 best_iteration = model.best_iteration_
test_predictions += model.predict_proba(x_test, num_iteration = best_iteration)[:, 1] / k_fold.n_splits
out_of_fold[valid_indices] = model.predict_proba(valid_features, num_iteration = best_iteration)[:, 1]
valid_score = model.best_score_['valid']['auc']
train_score = model.best_score_['train']['auc']
valid_scores.append(valid_score)
 train_scores.append(train_score)
 gc.enable()
del model, train_features, valid_features
gc.collect()<compute_test_metric>
from sklearn.linear_model import LinearRegression
import eli5
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
valid_auc = roc_auc_score(y_train, out_of_fold)
 valid_scores.append(valid_auc)
 train_scores.append(np.mean(train_scores))
fold_names = list(range(5))
fold_names.append('overall')
 metrics = pd.DataFrame({'fold': fold_names, 'train': train_scores, 'valid': valid_scores} )<save_to_csv>
log_re=np.log1p(train['revenue'])
 v1=TfidfVectorizer(sublinear_tf=True,analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
min_df=5)
 overview_text = v1.fit_transform(train['overview'].fillna(''))
linre=LinearRegression()
linre.fit(overview_text,log_re)
submission = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': test_predictions})
 submission.to_csv('submission.csv', index = False )<load_from_csv>
eli5.show_weights(linre, vec=v1, top=20, feature_filter=lambda x: x !='<BIAS>' )
def geometric_mean(x):
return np.exp(np.log(x[x>0] ).mean())
 def one_hot_encoder(df, nan_as_category = True):
original_columns = list(df.columns)
 categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)
 new_columns = [c for c in df.columns if c not in original_columns]
return df, new_columns
def application_train_test(num_rows = None, nan_as_category = False):
df = pd.read_csv('.. /input/application_train.csv', nrows= num_rows)
 test_df = pd.read_csv('.. /input/application_test.csv', nrows= num_rows)
 print("Train samples: {}, test samples: {}".format(len(df), len(test_df)))
 df = df.append(test_df ).reset_index()
df = df[df['CODE_GENDER'] != 'XNA']
docs = [_f for _f in df.columns if 'FLAG_DOC' in _f]
live = [_f for _f in df.columns if('FLAG_' in _f)&('FLAG_DOC' not in _f)&('_FLAG_' not in _f)]
df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)
 inc_by_org = df[['AMT_INCOME_TOTAL', 'ORGANIZATION_TYPE']].groupby('ORGANIZATION_TYPE' ).median() ['AMT_INCOME_TOTAL']
df['NEW_CREDIT_TO_ANNUITY_RATIO'] = df['AMT_CREDIT'] / df['AMT_ANNUITY']
df['NEW_CREDIT_TO_GOODS_RATIO'] = df['AMT_CREDIT'] / df['AMT_GOODS_PRICE']
df['NEW_DOC_IND_KURT'] = df[docs].kurtosis(axis=1)
 df['NEW_LIVE_IND_SUM'] = df[live].sum(axis=1)
 df['NEW_INC_PER_CHLD'] = df['AMT_INCOME_TOTAL'] /(1 + df['CNT_CHILDREN'])
 df['NEW_INC_BY_ORG'] = df['ORGANIZATION_TYPE'].map(inc_by_org)
 df['NEW_EMPLOY_TO_BIRTH_RATIO'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['NEW_EMPLOY_TO_BIRTH-18_RATIO'] = df['DAYS_EMPLOYED'] /(df['DAYS_BIRTH'] + 18*365)
 df['NEW_BIRTH_TO_EMPLOY_RATIO'] = df['DAYS_BIRTH'] /(1 + df['DAYS_EMPLOYED'])
 df['NEW_INCOME_TO_ANNUITY_RATIO'] = df['AMT_INCOME_TOTAL'] /(1 + df['AMT_ANNUITY'])
 df['NEW_ANNUITY_TO_INCOME_RATIO'] = df['AMT_ANNUITY'] /(1 + df['AMT_INCOME_TOTAL'])
 df['NEW_EXT_SOURCES_MEDIAN'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].median(axis=1, skipna=True)
 df['NEW_EXT_SOURCES_MEAN'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1, skipna=True)
 df['NEW_EXT_SOURCES_PROD'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].prod(axis=1, skipna=True, min_count=1)
 df['NEW_EXT_SOURCES_MAX'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].max(axis=1, skipna=True)
 df['NEW_EXT_SOURCES_MIN'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].min(axis=1, skipna=True)
 df['NEW_EXT_SOURCES_STD'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1, skipna=True)
 df['NEW_EXT_SOURCES_MAD'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mad(axis=1, skipna=True)
 df['NEW_EXT_SOURCES_GEO'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].apply(geometric_mean, axis=1)
 df['NEW_CAR_TO_BIRTH_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_BIRTH']
df['NEW_CAR_TO_EMPLOY_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_EMPLOYED']
df['NEW_PHONE_TO_BIRTH_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_BIRTH']
df['NEW_PHONE_TO_EMPLOYED_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_EMPLOYED']
df['NEW_CREDIT_TO_INCOME_RATIO'] = df['AMT_CREDIT'] / df['AMT_INCOME_TOTAL']
df['NEW_PAYMENT_RATE'] = df['AMT_ANNUITY'] / df['AMT_CREDIT']
df['NEW_INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']
df['NEW_INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']
for bin_feature in ['CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY']:
df[bin_feature], uniques = pd.factorize(df[bin_feature])
 df, cat_cols = one_hot_encoder(df, nan_as_category)
 dropcolum=['FLAG_DOCUMENT_2','FLAG_DOCUMENT_4',
'FLAG_DOCUMENT_5','FLAG_DOCUMENT_6','FLAG_DOCUMENT_7',
'FLAG_DOCUMENT_8','FLAG_DOCUMENT_9','FLAG_DOCUMENT_10',
'FLAG_DOCUMENT_11','FLAG_DOCUMENT_12','FLAG_DOCUMENT_13',
'FLAG_DOCUMENT_14','FLAG_DOCUMENT_15','FLAG_DOCUMENT_16',
'FLAG_DOCUMENT_17','FLAG_DOCUMENT_18','FLAG_DOCUMENT_19',
'FLAG_DOCUMENT_20','FLAG_DOCUMENT_21']
df= df.drop(dropcolum,axis=1)
 del test_df
gc.collect()
return df
def bureau_and_balance(num_rows = None, nan_as_category = True):
bureau = pd.read_csv('.. /input/bureau.csv', nrows = num_rows)
 bb = pd.read_csv('.. /input/bureau_balance.csv', nrows = num_rows)
 bb, bb_cat = one_hot_encoder(bb, nan_as_category)
 bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category)
 bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size']}
for col in bb_cat:
bb_aggregations[col] = ['mean']
bb_agg = bb.groupby('SK_ID_BUREAU' ).agg(bb_aggregations)
 bb_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist() ])
 bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')
 bureau.drop(['SK_ID_BUREAU'], axis=1, inplace= True)
 del bb, bb_agg
gc.collect()
num_aggregations = {
'DAYS_CREDIT': [ 'mean', 'var'],
'DAYS_CREDIT_ENDDATE': [ 'mean'],
'DAYS_CREDIT_UPDATE': ['mean'],
'CREDIT_DAY_OVERDUE': ['mean'],
'AMT_CREDIT_MAX_OVERDUE': ['mean'],
'AMT_CREDIT_SUM': [ 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': [ 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['mean'],
'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],
'AMT_ANNUITY': ['max', 'mean'],
'CNT_CREDIT_PROLONG': ['sum'],
'MONTHS_BALANCE_MIN': ['min'],
'MONTHS_BALANCE_MAX': ['max'],
'MONTHS_BALANCE_SIZE': ['mean', 'sum']
}
cat_aggregations = {}
for cat in bureau_cat: cat_aggregations[cat] = ['mean']
for cat in bb_cat: cat_aggregations[cat + "_MEAN"] = ['mean']
bureau_agg = bureau.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 bureau_agg.columns = pd.Index(['BURO_' + e[0] + "_" + e[1].upper() for e in bureau_agg.columns.tolist() ])
 active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]
active_agg = active.groupby('SK_ID_CURR' ).agg(num_aggregations)
 active_agg.columns = pd.Index(['ACTIVE_' + e[0] + "_" + e[1].upper() for e in active_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(active_agg, how='left', on='SK_ID_CURR')
 del active, active_agg
gc.collect()
closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]
closed_agg = closed.groupby('SK_ID_CURR' ).agg(num_aggregations)
 closed_agg.columns = pd.Index(['CLOSED_' + e[0] + "_" + e[1].upper() for e in closed_agg.columns.tolist() ])
 bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')
 del closed, closed_agg, bureau
gc.collect()
return bureau_agg
def previous_applications(num_rows = None, nan_as_category = True):
prev = pd.read_csv('.. /input/previous_application.csv', nrows = num_rows)
 prev, cat_cols = one_hot_encoder(prev, nan_as_category= True)
 prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)
 prev['APP_CREDIT_PERC'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']
num_aggregations = {
'AMT_ANNUITY': [ 'max', 'mean'],
'AMT_APPLICATION': [ 'max','mean'],
'AMT_CREDIT': [ 'max', 'mean'],
'APP_CREDIT_PERC': [ 'max', 'mean'],
'AMT_DOWN_PAYMENT': [ 'max', 'mean'],
'AMT_GOODS_PRICE': [ 'max', 'mean'],
'HOUR_APPR_PROCESS_START': [ 'max', 'mean'],
'RATE_DOWN_PAYMENT': [ 'max', 'mean'],
'DAYS_DECISION': [ 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
}
cat_aggregations = {}
for cat in cat_cols:
cat_aggregations[cat] = ['mean']
prev_agg = prev.groupby('SK_ID_CURR' ).agg({**num_aggregations, **cat_aggregations})
 prev_agg.columns = pd.Index(['PREV_' + e[0] + "_" + e[1].upper() for e in prev_agg.columns.tolist() ])
 approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]
approved_agg = approved.groupby('SK_ID_CURR' ).agg(num_aggregations)
 approved_agg.columns = pd.Index(['APPROVED_' + e[0] + "_" + e[1].upper() for e in approved_agg.columns.tolist() ])
 prev_agg = prev_agg.join(approved_agg, how='left', on='SK_ID_CURR')
 refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]
refused_agg = refused.groupby('SK_ID_CURR' ).agg(num_aggregations)
 refused_agg.columns = pd.Index(['REFUSED_' + e[0] + "_" + e[1].upper() for e in refused_agg.columns.tolist() ])
 prev_agg = prev_agg.join(refused_agg, how='left', on='SK_ID_CURR')
 del refused, refused_agg, approved, approved_agg, prev
gc.collect()
return prev_agg
def pos_cash(num_rows = None, nan_as_category = True):
pos = pd.read_csv('.. /input/POS_CASH_balance.csv', nrows = num_rows)
 pos, cat_cols = one_hot_encoder(pos, nan_as_category= True)
 aggregations = {
'MONTHS_BALANCE': ['max', 'mean', 'size'],
'SK_DPD': ['max', 'mean'],
'SK_DPD_DEF': ['max', 'mean']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
pos_agg = pos.groupby('SK_ID_CURR' ).agg(aggregations)
 pos_agg.columns = pd.Index(['POS_' + e[0] + "_" + e[1].upper() for e in pos_agg.columns.tolist() ])
 pos_agg['POS_COUNT'] = pos.groupby('SK_ID_CURR' ).size()
del pos
gc.collect()
return pos_agg
def installments_payments(num_rows = None, nan_as_category = True):
ins = pd.read_csv('.. /input/installments_payments.csv', nrows = num_rows)
 ins, cat_cols = one_hot_encoder(ins, nan_as_category= True)
 ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']
ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']
ins['DPD'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']
ins['DBD'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']
ins['DPD'] = ins['DPD'].apply(lambda x: x if x > 0 else 0)
 ins['DBD'] = ins['DBD'].apply(lambda x: x if x > 0 else 0)
 aggregations = {
'NUM_INSTALMENT_VERSION': ['nunique'],
'DPD': ['max', 'mean', 'sum'],
'DBD': ['max', 'mean', 'sum'],
'PAYMENT_PERC': [ 'mean', 'var'],
'PAYMENT_DIFF': [ 'mean', 'var'],
'AMT_INSTALMENT': ['max', 'mean', 'sum'],
'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],
'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum']
}
for cat in cat_cols:
aggregations[cat] = ['mean']
ins_agg = ins.groupby('SK_ID_CURR' ).agg(aggregations)
 ins_agg.columns = pd.Index(['INSTAL_' + e[0] + "_" + e[1].upper() for e in ins_agg.columns.tolist() ])
 ins_agg['INSTAL_COUNT'] = ins.groupby('SK_ID_CURR' ).size()
del ins
gc.collect()
return ins_agg
def credit_card_balance(num_rows = None, nan_as_category = True):
cc = pd.read_csv('.. /input/credit_card_balance.csv', nrows = num_rows)
 cc, cat_cols = one_hot_encoder(cc, nan_as_category= True)
 cc.drop(['SK_ID_PREV'], axis= 1, inplace = True)
 cc_agg = cc.groupby('SK_ID_CURR' ).agg([ 'max', 'mean', 'sum', 'var'])
 cc_agg.columns = pd.Index(['CC_' + e[0] + "_" + e[1].upper() for e in cc_agg.columns.tolist() ])
 cc_agg['CC_COUNT'] = cc.groupby('SK_ID_CURR' ).size()
del cc
gc.collect()
return cc_agg 
<categorify>
new_data['crew_0']=new_data['crew'].fillna('' ).str.replace(',','' ).str.replace('}','' ).str.findall(''gender\S?'\s?:\s?\S?(\d+)\s?\S?' ).apply(lambda x:x.count('0'))
new_data['crew_1']=new_data['crew'].fillna('' ).str.replace(',','' ).str.replace('}','' ).str.findall(''gender\S?'\s?:\s?\S?(\d+)\s?\S?' ).apply(lambda x:x.count('1'))
new_data['crew_2']=new_data['crew'].fillna('' ).str.replace(',','' ).str.replace('}','' ).str.findall(''gender\S?'\s?:\s?\S?(\d+)\s?\S?' ).apply(lambda x:x.count('2'))
def kfold_lightgbm(train_df, train_target, test_df, num_folds, stratified=False, debug=False):
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))
if stratified:
folds = StratifiedKFold(n_splits=num_folds, shuffle=True, random_state=47)
 else:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=47)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feat_importance = pd.DataFrame()
scores = []
models = []
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df, train_target)) :
train_x, train_y = train_df.iloc[train_idx], train_target.iloc[train_idx]
valid_x, valid_y = train_df.iloc[valid_idx], train_target.iloc[valid_idx]
clf = LGBMClassifier(
 nthread=4,
n_estimators=10000,
learning_rate=0.02,
num_leaves=32,
colsample_bytree=0.9497036,
subsample=0.8715623,
max_depth=8,
reg_alpha=0.04,
reg_lambda=0.073,
min_split_gain=0.0222415,
min_child_weight=40,
silent=-1,
verbose=-1, 
)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
eval_metric= 'auc', verbose= 1000, early_stopping_rounds= 300)
 oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test_df, num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = test_df.columns.values
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["shap_values"] = abs(shap.TreeExplainer(clf ).shap_values(valid_x)[:,:test_df.shape[1]] ).mean(axis=0 ).T
fold_importance_df["fold"] = n_fold + 1
feat_importance = pd.concat([feat_importance, fold_importance_df], axis=0)
 scores.append(roc_auc_score(valid_y, oof_preds[valid_idx]))
print('Fold %2d AUC : %.6f' %(n_fold + 1, scores[n_fold]))
models.append(clf)
 del clf, train_x, train_y, valid_x, valid_y, fold_importance_df
gc.collect()
score = roc_auc_score(train_target, oof_preds)
 print('Full AUC score %.6f' % score)
 print('Mean AUC score %.6f' % np.mean(scores))
if not debug:
pd.DataFrame(oof_preds ).to_csv("lgb{:03}_{:.5f}_train_oof.csv".format(test_df.shape[1], score), index=False)
 sub_df = pd.read_csv('.. /input/sample_submission.csv')
 sub_df['TARGET'] = sub_preds
sub_df.to_csv("lgb{:03}_{:.5f}.csv".format(test_df.shape[1], score), index= False)
 display_shapley_values(feat_importance)
 return feat_importance, models, scores
def display_importances(feat_importance):
best_features = feat_importance[["feature", "importance"]].groupby("feature")["importance"].agg(['mean', 'std'])\ 
.sort_values(by="mean", ascending=False ).head(40 ).reset_index()
best_features.columns = ["feature", "mean importance", "err"]
plt.figure(figsize=(8, 10))
sns.barplot(x="mean importance", y="feature", xerr=best_features['err'], data=best_features)
 plt.title('LightGBM Features(avg over folds)')
 plt.tight_layout()
plt.show()
def display_shapley_values(feat_importance):
best_features = feat_importance[["feature", "shap_values"]].groupby("feature")["shap_values"].agg(['mean', 'std'])\ 
.sort_values(by="mean", ascending=False ).head(40 ).reset_index()
best_features.columns = ["feature", "mean shapley values", "err"]
plt.figure(figsize=(8, 10))
sns.barplot(x="mean shapley values", y="feature", xerr=best_features['err'], data=best_features)
 plt.title('LightGBM shapley values(avg over folds)')
 plt.tight_layout()
plt.show() 
<categorify>
for i in d:
new_data['crew_depart_is_'+i[0]]=new_data['crew'].fillna('' ).apply(lambda x:1 if i[0] in x else 0)
 for i in dd:
new_data['crew_job_is_'+i[0]]=new_data['crew'].fillna('' ).apply(lambda x:1 if i[0] in x else 0 )
%%time
debug = False
num_rows = 10000 if debug else None
scores = {}
df = application_train_test(num_rows)
 with timer("Process bureau and bureau_balance"):
bureau = bureau_and_balance(num_rows)
 print("Bureau df shape:", bureau.shape)
 df = df.join(bureau, how='left', on='SK_ID_CURR')
 del bureau
gc.collect()
with timer("Process previous_applications"):
prev = previous_applications(num_rows)
 print("Previous applications df shape:", prev.shape)
 df = df.join(prev, how='left', on='SK_ID_CURR')
 del prev
gc.collect()
with timer("Process POS-CASH balance"):
pos = pos_cash(num_rows)
 print("Pos-cash balance df shape:", pos.shape)
 df = df.join(pos, how='left', on='SK_ID_CURR')
 del pos
gc.collect()
with timer("Process installments payments"):
ins = installments_payments(num_rows)
 print("Installments payments df shape:", ins.shape)
 df = df.join(ins, how='left', on='SK_ID_CURR')
 del ins
gc.collect()
with timer("Process credit card balance"):
cc = credit_card_balance(num_rows)
 print("Credit card balance df shape:", cc.shape)
 df = df.join(cc, how='left', on='SK_ID_CURR')
 del cc
gc.collect()
with timer("Save df"):
df.to_csv('merged_df.csv.gz', compression='gzip', index=False)
 with timer("Divide in training and test data"):
feats = [f for f in df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]
train_df = df[df['TARGET'].notnull() ][feats]
train_target = df[df['TARGET'].notnull() ]['TARGET']
test_df = df[df['TARGET'].isnull() ][feats]
del df
gc.collect()<compute_train_metric>
new_data['crew']=new_data['crew'].str.replace(',','' ).str.replace('}','' ).str.replace(r''gender'\s?\S?:\s?\S?\s+'id'\s?:\s?\d+','' ).str.findall(r''department'\s?\S?:\s+\S?(\S+)\S?\s?\S?\s+'job\S?'\s?:\s?'(\D+)\S\s?\S?'name\S?'\s?:\s?'(\D+)'\s?\S?'profile_path\S?':\s+'?(\S+)\S?\s?\s?\S?' )
%%time
feat_importance, models, scores = kfold_lightgbm(train_df, train_target, test_df, num_folds=5, stratified=False, debug=debug )<compute_test_metric>
new_data['len_crew']=new_data['crew'].fillna('1' ).apply(lambda x:len(x))
def inv_logit(p): return np.exp(p)/(1 + np.exp(p))
base_value = shap_values[0,-1]
output = base_value + np.sum(shap_values[0,:-1])
 print('Log-odds output:', output, ' Logistic output:', inv_logit(output))<compute_train_metric>
new_train=new_data.loc[np.array(train.index)]
percentile = 0.15
best_features = feat_importance[["feature", "shap_values"]].groupby("feature")["shap_values"].agg(['mean'])\ 
.sort_values(by="mean", ascending=False ).reset_index()
best_features = best_features[:int(best_features.shape[0]*percentile)]["feature"].values
print("
with timer("Run LightGBM with kfold"):
train_df = train_df[best_features]
test_df = test_df[best_features]
feat_importance, models, scores = kfold_lightgbm(train_df, train_target, test_df, num_folds=5, stratified=False, debug=debug )<set_options>
new_train['production_countries']=new_train['production_countries'].fillna('' )
plt.rcParams['font.size'] = 18
plt.style.use('fivethirtyeight')
 %matplotlib inline<load_from_csv>
new_train['production_countries']=new_train['production_countries'].apply(lambda x:'_'.join(x))
random = pd.read_csv('.. /input/home-credit-model-tuning/random_search_simple.csv' ).sort_values('score', ascending = False ).reset_index()
opt = pd.read_csv('.. /input/home-credit-model-tuning/bayesian_trials_simple.csv' ).sort_values('score', ascending = False ).reset_index()
print('Best score from random search: {:.5f} found on iteration: {}.'.format(random.loc[0, 'score'], random.loc[0, 'iteration']))
print('Best score from bayesian optimization: {:.5f} found on iteration: {}.'.format(opt.loc[0, 'score'], opt.loc[0, 'iteration']))<compute_test_metric>
count=new_train['production_countries'].value_counts()
count=count[count>5]
keys = []
for key, value in ast.literal_eval(random.loc[0, 'hyperparameters'] ).items() :
print(f'{key}: {value}')
 keys.append(key )<compute_test_metric>
bud=new_train.groupby('production_countries' ).budget.mean()
for key in keys:
print('{}: {}'.format(key, ast.literal_eval(opt.loc[0, 'hyperparameters'])[key]))<find_best_params>
bud=bud.sort_values(ascending=False)[:10]
random['set'] = 'random'
scores = random[['score', 'iteration', 'set']]
opt['set'] = 'opt'
scores = scores.append(opt[['set', 'iteration', 'score']], sort = True)
 scores.head()<groupby>
rev=new_train.groupby('production_countries' ).revenue.mean()
scores.groupby('set')['score'].agg(['mean', 'max', 'min', 'std', 'count'] )<train_model>
rev=rev.sort_values(ascending=False)[:10]
random_fit = np.polyfit(random['iteration'], random['score'], 1)
 print('Random search slope: {:.8f}'.format(random_fit[0]))<train_model>
new_train['production_countries'].loc[new_train['production_countries']=='ET']='Ethiopia'
along_co=new_train[new_train['production_countries'].apply(lambda x:1 if len(x)==2 else 0)==1]
along_co.head()
opt_fit = np.polyfit(opt['iteration'], opt['score'], 1)
 print('opt search slope: {:.8f}'.format(opt_fit[0]))<feature_engineering>
count_rev=along_co[['production_countries','revenue']]
opt_fit[0] / random_fit[0]<train_model>
dd=count_rev['production_countries'].unique()
mn=pd.DataFrame(dd,columns=['address'])
 mn.head()
print('After 10,000 iterations, the random score is: {:.5f}.'.format(
 random_fit[0] * 1e5 + random_fit[1]))<train_model>
import geopandas as ge
print('After 10,000 iterations, the bayesian score is: {:.5f}.'.format(
 opt_fit[0] * 1e5 + opt_fit[1]))<prepare_output>
world = ge.read_file(ge.datasets.get_path('naturalearth_lowres'))
def process(results):

 results = results.copy()
results['hyperparameters'] = results['hyperparameters'].map(ast.literal_eval)
 results = results.sort_values('score', ascending = False ).reset_index(drop = True)
 hyp_df = pd.DataFrame(columns = list(results.loc[0, 'hyperparameters'].keys()))
for i, hyp in enumerate(results['hyperparameters']):
hyp_df = hyp_df.append(pd.DataFrame(hyp, index = [0]),
ignore_index = True, sort= True)
 hyp_df['iteration'] = results['iteration']
hyp_df['score'] = results['score']
return hyp_df<define_variables>
from mpl_toolkits.axes_grid1 import make_axes_locatable
random_hyp = process(random)
 opt_hyp = process(opt)
 random_hyp.head()<define_search_space>
dd=count_rev['production_countries'].value_counts()
mn['values']=dd.values
mn.head()
param_grid = {
'is_unbalance': [True, False],
'boosting_type': ['gbdt', 'goss', 'dart'],
'num_leaves': list(range(20, 150)) ,
'learning_rate': list(np.logspace(np.log10(0.005), np.log10(0.5), base = 10, num = 1000)) ,
'subsample_for_bin': list(range(20000, 300000, 20000)) ,
'min_child_samples': list(range(20, 500, 5)) ,
'reg_alpha': list(np.linspace(0, 1)) ,
'reg_lambda': list(np.linspace(0, 1)) ,
'colsample_bytree': list(np.linspace(0.6, 1, 10)) ,
'subsample': list(np.linspace(0.5, 1, 100))
}<find_best_params>
mn['address'][0]='United States'
mn['address'][2]='Korea'
mn['address'][1]='India'
mn['address'][3]='Serbia'
mn['address'][4]='United Kingdom'
mn['address'][5]='France'
mn['address'][6]='New Zealand'
mn['address'][7]='Italy'
mn['address'][8]='Belgium'
mn['address'][9]='Czech Rep.'
mn['address'][11]='Russia'
mn['address'][12]='Spain'
mn['address'][13]='Turkey'
mn['address'][14]='China'
mn['address'][15]='Canada'
mn['address'][16]='Australia'
mn['address'][17]='Iran'
mn['address'][18]='Japan'
mn['address'][19]='Sweden'
mn['address'][20]='Philippines'
mn['address'][21]='Brazil'
mn['address'][22]='Netherlands'
mn['address'][23]='Ireland'
mn['address'][24]='Mexico'
mn['address'][25]='Germany'
mn['address'][26]='South Africa'
mn['address'][27]='Finland'
mn['address'][28]='Pakistan'
mn['address'][29]='Norway'
mn['address'][30]='Bulgaria'
mn['address'][31]='Ukraine'
mn['address'][32]='Denmark'
mn['address'][33]='Romania'
mn['address'][35]='Hungary'
mn['address'][36]='Chile'
mn['address'][37]='Indonesia'
mn['address'][38]='Poland'
best_random_hyp = random_hyp.loc[0, :]
best_opt_hyp = opt_hyp.loc[0, :]<groupby>
mn=mn.drop(10)
 mn=mn.drop(34 )
random_hyp.groupby('boosting_type')['score'].agg(['mean', 'max', 'min', 'std', 'count'] )<groupby>
mn['geometry']='unknow'
opt_hyp.groupby('boosting_type')['score'].agg(['mean', 'max', 'min', 'std', 'count'] )<groupby>
for i in range(39):
if i==10 or i==34:
continue;
d=world[world['name']==mn['address'][i]]['geometry'].values[0]
mn['geometry'][i]=d
random_hyp.groupby('is_unbalance')['score'].agg(['mean', 'max', 'min', 'std', 'count'] )<groupby>
TS = train.loc[:,["original_title","release_date","budget","runtime","revenue"]]
TS.dropna()
TS.release_date = pd.to_datetime(TS.release_date)
 TS.loc[:,"Year"] = TS["release_date"].dt.year
TS.loc[:,"Month"] = TS["release_date"].dt.month
TS = TS[TS.Year<2018]
opt_hyp.groupby('is_unbalance')['score'].agg(['mean', 'max', 'min', 'std', 'count'] )<prepare_output>
top3 = train.sort_values(by='popularity',ascending=False)[:10]
id3=top3[['title','poster_path','revenue']]
random_hyp['set'] = 'Random Search'
opt_hyp['set'] = 'Bayesian'
hyp = random_hyp.append(opt_hyp, ignore_index = True, sort = True)
 hyp.head()<set_options>
old_title=new_data['title'].value_counts()
plt.rcParams['axes.labelpad'] = 12<data_type_conversions>
a=old_title[old_title.values==3].index
b=old_title[old_title==2].index
random_hyp['n_estimators'] = random_hyp['n_estimators'].astype(np.int32)
 random_hyp.corr() ['score']<data_type_conversions>
new_data['fan_pai_2']=new_data['title'].apply(lambda x:1 if x in a else 0)
 new_data['pan_pai_3']=new_data['title'].apply(lambda x:1 if x in b else 0 )
opt_hyp['n_estimators'] = opt_hyp['n_estimators'].astype(np.int32)
 opt_hyp.corr() ['score']<categorify>
new_data.loc[new_data['title'].fillna('un' ).str.contains('Planet of the Apes')]
hyp = hyp.drop(columns = ['metric', 'set', 'verbose'])
 hyp['n_estimators'] = hyp['n_estimators'].astype(np.int32)
 hyp['min_child_samples'] = hyp['min_child_samples'].astype(np.int32)
 hyp['num_leaves'] = hyp['num_leaves'].astype(np.int32)
 hyp['subsample_for_bin'] = hyp['subsample_for_bin'].astype(np.int32)
 hyp = pd.get_dummies(hyp)
 train_labels = hyp.pop('score')
 train = np.array(hyp.copy() )<train_model>
top10 = train.sort_values(by='revenue',ascending=False)[:10]
id10=top10[['title','poster_path','revenue']]
lr = LinearRegression()
lr.fit(train, train_labels )<import_modules>
title_count=train['title'].value_counts()
len(title_count[title_count.values!=1] )
import lightgbm as lgb<load_from_csv>
train[train['title'].str.contains('Furious')]['title']
train = pd.read_csv('.. /input/home-credit-simple-featuers/simple_features_train.csv')
 print('Full Training Features Shape: ', train.shape)
 test = pd.read_csv('.. /input/home-credit-simple-featuers/simple_features_test.csv')
 print('Full Testing Features Shape: ', test.shape )<data_type_conversions>
test[test['title'].fillna('Unknow' ).str.contains('Furious')]['title']
train_labels = np.array(train['TARGET'].astype(np.int32)).reshape(( -1,))
train = train.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test_ids = list(test['SK_ID_CURR'])
 test = test.drop(columns = ['SK_ID_CURR'] )<train_model>
d.remove(d[7] )
random_best = ast.literal_eval(random.loc[0, 'hyperparameters'])
 rmodel = lgb.LGBMClassifier(**random_best)
 rmodel.fit(train, train_labels )<save_to_csv>
for i in d :
new_data['crew_contains_'+i[0][2]]=new_data['crew'].fillna('' ).apply(lambda x:1 if i[0][2] in x else 0)
rpreds = rmodel.predict_proba(test)[:, 1]
rsub = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': rpreds})
 rsub.to_csv('submission_random_search.csv', index = False )<train_model>
mon_is=[1,6,7,9,10,12]
for i in mon_is:
new_data['mon_is_'+str(i)]=new_data['release_month'].apply(lambda x:1 if x==i else 0 )
bayes_best = ast.literal_eval(opt.loc[0, 'hyperparameters'])
 bmodel = lgb.LGBMClassifier(**bayes_best)
 bmodel.fit(train, train_labels )<save_to_csv>
new_data['spoken_languages_count']=new_data['spoken_languages'].apply(lambda x:len(x))
bpreds = bmodel.predict_proba(test)[:, 1]
bsub = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': bpreds})
 bsub.to_csv('submission_bayesian_optimization.csv', index = False )<create_dataframe>
new_data.loc[train.index].groupby('spoken_languages_count' ).revenue.median()
random_fi = pd.DataFrame({'feature': features, 'importance': rmodel.feature_importances_})
 bayes_fi = pd.DataFrame({'feature': features, 'importance': bmodel.feature_importances_} )<train_model>
status_du=pd.get_dummies(new_data['status'] )
random.loc[0, 'hyperparameters']<import_modules>
new_data=pd.concat([new_data,status_du],axis=1)
import matplotlib.pyplot as plt
import lightgbm as lgb
import gc
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import confusion_matrix,precision_recall_curve,auc,roc_auc_score,roc_curve,recall_score,classification_report
from sklearn.preprocessing import LabelEncoder<load_from_csv>
new_data=new_data.drop(['status'],axis=1)
 new_data.head()
app_train = pd.read_csv('.. /input/application_train.csv')
 print('Training data shape: ', app_train.shape)
 app_train.head()<load_from_csv>
new_data=new_data.drop(['original_language'],axis=1 )
app_test = pd.read_csv('.. /input/application_test.csv')
 print('Testing data shape: ', app_test.shape)
 app_test.head()<count_values>
sum(new_data['geres_name'].isna() )
app_train['TARGET'].value_counts()
print('The proportion of label 1 is %.2f' %(sum(app_train['TARGET']==1)/app_train.shape[0]*100), '%' )<count_missing_values>
new_data['geres_name']=new_data['geres_name'].fillna('Unknow' )
def missing_values_table(df):
mis_val = df.isnull().sum()
mis_val_percent = df.isnull().sum() * 100 / df.shape[0]
mis_val_table = pd.concat([mis_val, mis_val_percent], axis = 1)
 mis_val_table_rename_columns = mis_val_table.rename(columns = {0: 'Missing Values', 1: 'Percentage'})
 mis_val_table_rename_columns = mis_val_table_rename_columns[
mis_val_table_rename_columns.iloc[:, 1]!=0].sort_values('Percentage', ascending=False ).round(1)
 print('The total dataframe has ' + str(df.shape[1])+ ' columns')
 print('There are ' + str(mis_val_table_rename_columns.shape[0])+ ' columns that have missing values.')
 return mis_val_table_rename_columns<count_missing_values>
new_data['geres_name_count']=new_data['geres_name'].apply(lambda x:len(x))
missing_values = missing_values_table(app_train)
 missing_values.head(20 )<count_values>
ge_name_count_rev=new_data.loc[train.index].groupby(['geres_name_count'] ).revenue.median().sort_values()
app_train.dtypes.value_counts()<count_unique_values>
new_data=new_data.drop('homepage_end',axis=1 )
app_train.select_dtypes('object' ).apply(pd.Series.nunique, axis=0 )<count_values>
new_data['original_and_new']=new_data['original_and_new'].apply(lambda x: 1 if x else 0 )
app_test.dtypes.value_counts()<categorify>
list_geres_name=['Comedy','Thriller','Action','Drama','Romance']
app_train = pd.get_dummies(app_train)
 app_test = pd.get_dummies(app_test)
 print('Training Features shape: ', app_train.shape)
 print('Testing Features shape: ', app_test.shape )<train_model>
for i in list_geres_name:
new_data['geres_name'+'_'+i]=new_data['geres_name'].apply(lambda x: 1 if i in x else 0 )
train_labels = app_train['TARGET']
app_train, app_test = app_train.align(app_test, join = 'inner', axis = 1)
 app_train['TARGET'] = train_labels
print('Training Features shape: ', app_train.shape)
 print('Testing Features shape: ', app_test.shape )<train_model>
new_data=new_data.drop('geres_name',axis=1 )
app_train['DAYS_EMPLOYED_ANOM'] = app_train['DAYS_EMPLOYED'] == 365243
app_train['DAYS_EMPLOYED'].replace({365243: np.nan}, inplace = True)
 app_test['DAYS_EMPLOYED_ANOM'] = app_test['DAYS_EMPLOYED'] == 365243
app_test['DAYS_EMPLOYED'].replace({365243: np.nan}, inplace = True)
 print('Training Features shape: ', app_train.shape)
 print('Testing Features shape: ', app_test.shape )<feature_engineering>
new_data['production_countries']=new_data['production_countries'].fillna('QQ' )
app_train_domain = app_train.copy()
app_test_domain = app_test.copy()
app_train_domain['CREDIT_INCOME_PERCENT'] = app_train_domain['AMT_CREDIT'] / app_train_domain['AMT_INCOME_TOTAL']
app_train_domain['ANNUITY_INCOME_PERCENT'] = app_train_domain['AMT_ANNUITY'] / app_train_domain['AMT_INCOME_TOTAL']
app_train_domain['CREDIT_TERM'] = app_train_domain['AMT_ANNUITY'] / app_train_domain['AMT_CREDIT']
app_train_domain['DAYS_EMPLOYED_PERCENT'] = app_train_domain['DAYS_EMPLOYED'] / app_train_domain['DAYS_BIRTH']
app_test_domain['CREDIT_INCOME_PERCENT'] = app_test_domain['AMT_CREDIT'] / app_test_domain['AMT_INCOME_TOTAL']
app_test_domain['ANNUITY_INCOME_PERCENT'] = app_test_domain['AMT_ANNUITY'] / app_test_domain['AMT_INCOME_TOTAL']
app_test_domain['CREDIT_TERM'] = app_test_domain['AMT_ANNUITY'] / app_test_domain['AMT_CREDIT']
app_test_domain['DAYS_EMPLOYED_PERCENT'] = app_test_domain['DAYS_EMPLOYED'] / app_test_domain['DAYS_BIRTH']
print('Domain Training Features shape: ', app_train_domain.shape)
 print('Domain Testing Features shape: ', app_test_domain.shape )<load_from_csv>
new_data['production_countries'].loc[new_data['production_countries']=='Unknow']['production_countries']='QQ'
bureau = pd.read_csv('.. /input/bureau.csv')
 bureau.head()<groupby>
list_pro_coun=list(new_data['production_countries'])
 d = Counter([j for i in list_pro_coun for j in i] ).most_common(5)
 d
previous_loan_counts = bureau.groupby('SK_ID_CURR', as_index=False)['SK_ID_BUREAU'].count().rename(columns = {'SK_ID_BUREAU': 'previous_loan_counts'})
 previous_loan_counts.head() 
<create_dataframe>
for i in d:
new_data['production_cou_name'+'_'+i[0]]=new_data['production_countries'].apply(lambda x: 1 if i[0] in x else 0 )
def agg_numeric(df, group_var, df_name):

 for col in df:
if col != group_var and 'SK_ID' in col:
df = df.drop(columns = col)
 group_ids = df[group_var]
numeric_df = df.select_dtypes('number')
 numeric_df[group_var] = group_ids
agg = numeric_df.groupby(group_var ).agg(['count', 'mean', 'max', 'min', 'sum'] ).reset_index()
columns = [group_var]
for var in agg.columns.levels[0]:
if var != group_var:
for stat in agg.columns.levels[1][:-1]:
columns.append('%s_%s_%s' %(df_name, var, stat))
agg.columns = columns
return agg
bureau_agg = agg_numeric(bureau.drop(columns = ['SK_ID_BUREAU']), group_var = 'SK_ID_CURR', df_name = 'bureau')
 bureau_agg.head()<create_dataframe>
new_data['pro_country_count']=new_data['production_companies'].apply(lambda x:len(x))
def count_categorical(df, group_var, df_name):

 categorical = pd.get_dummies(df.select_dtypes('object'))
categorical[group_var] = df[group_var]
categorical = categorical.groupby(group_var ).agg(['sum', 'mean'])
 column_names = []
for var in categorical.columns.levels[0]:
for stat in ['count', 'count_norm']:
column_names.append('%s_%s_%s' %(df_name, var, stat))
categorical.columns = column_names
return categorical
bureau_counts = count_categorical(bureau, group_var = 'SK_ID_CURR', df_name = 'bureau')
 bureau_counts.head()<merge>
pro_count_rev=new_data.loc[train.index].groupby('pro_country_count' ).revenue.median()
train = app_train_domain.merge(previous_loan_counts, on = 'SK_ID_CURR', how = 'left')
 train['previous_loan_counts'] = train['previous_loan_counts'].fillna(0)
 test = app_test_domain.merge(previous_loan_counts, on = 'SK_ID_CURR', how = 'left')
 test['previous_loan_counts'] = test['previous_loan_counts'].fillna(0 )<merge>
new_data=new_data.drop('production_countries',axis=1 )
train = train.merge(bureau_agg, on = 'SK_ID_CURR', how = 'left')
 test = test.merge(bureau_agg, on = 'SK_ID_CURR', how = 'left' )<merge>
new_data['production_companies'].loc[new_data['production_companies']=='unknow']='u'
train = train.merge(bureau_counts, on = 'SK_ID_CURR', how = 'left')
 test = test.merge(bureau_counts, on = 'SK_ID_CURR', how = 'left' )<train_model>
d.remove(( 'u',414))
print('Before align train.shape: ', train.shape)
 print('Before align test.shape: ', test.shape)
 train_labels = train['TARGET']
train, test = train.align(test, join = 'inner', axis = 1)
 train['TARGET'] = train_labels
print('After align train.shape: ', train.shape)
 print('After align test.shape: ', test.shape )<load_from_csv>
dd=[]
for i in d:
dd.append(i[0] )
bureau_balance = pd.read_csv('.. /input/bureau_balance.csv')
 bureau_balance.head()<categorify>
dd=[]
for i in d:
dd.append(i[0] )
bureau_balance_agg = agg_numeric(bureau_balance, group_var = 'SK_ID_BUREAU', df_name = 'bureau_balance')
 bureau_balance_agg.head()<groupby>
new_data=new_data.drop('spoken_languages',axis=1 )
bureau_balance_counts = count_categorical(bureau_balance, group_var = 'SK_ID_BUREAU', df_name = 'bureau_balance')
 bureau_balance_counts.head()<merge>
new_data['runtime'].loc[new_data['runtime']==0]=1
new_data['time_budget']=new_data['budget'].fillna(0)/new_data['runtime'].fillna(1 )
bureau_by_loan = bureau_balance_agg.merge(bureau_balance_counts, right_index = True, left_on = 'SK_ID_BUREAU', how = 'outer' )<merge>
new_data=new_data.drop('production_companies',axis=1 )
bureau_by_loan = bureau_by_loan.merge(bureau[['SK_ID_BUREAU', 'SK_ID_CURR']], on = 'SK_ID_BUREAU', how = 'left' )<drop_column>
new_data=new_data.drop('crew',axis=1 )
bureau_balance_by_client = agg_numeric(bureau_by_loan.drop(columns = ['SK_ID_BUREAU']), group_var = 'SK_ID_CURR', df_name = 'client')
 bureau_balance_by_client.head()<merge>
for i in list1:
new_data['title_is'+i]=new_data['title'].fillna('' ).apply(lambda x: 1 if i in x else 0 )
train = train.merge(bureau_balance_by_client, on = 'SK_ID_CURR', how = 'left')
 test = test.merge(bureau_balance_by_client, on = 'SK_ID_CURR', how = 'left' )<train_model>
for col in ['tagline', 'overview','title']:
new_data['len_' + col] =new_data[col].fillna('' ).apply(lambda x: len(x))
new_data['words_' + col] = new_data[col].fillna('' ).apply(lambda x: len(x.split(' ')))
 new_data=new_data.drop(col,axis=1 )
print('Before align train.shape: ', train.shape)
 print('Before align test.shape: ', test.shape)
 train_labels = train['TARGET']
train, test = train.align(test, join = 'inner', axis = 1)
 train['TARGET'] = train_labels
print('After align train.shape: ', train.shape)
 print('After align test.shape: ', test.shape )<set_options>
new_data['budget_popularity']=new_data['budget']*1.0/new_data['popularity']
gc.enable()
del app_train, app_test, app_train_domain, app_test_domain, bureau, bureau_balance, bureau_agg, bureau_counts, bureau_balance_agg, bureau_balance_counts, bureau_by_loan, bureau_balance_by_client
gc.collect()<categorify>
a=new_data.groupby(['release_year'] ).release_month.value_counts()
b=new_data.groupby(['release_year','release_month'] ).release_day.value_counts()
new_data[(new_data['release_year']==2017)&(new_data['release_day']==30)]
missing_train = missing_values_table(train)
 missing_train.head(10 )<count_missing_values>
new_data['_releaseYear_popularity_ratio'] = new_data['release_year'] / new_data['popularity']
new_data['_releaseYear_popularity_ratio2'] = new_data['popularity'] / new_data['release_year']
new_data['runtime_to_mean_year'] = new_data['runtime'] / new_data.groupby("release_year")["runtime"].transform('mean')
 new_data['popularity_to_mean_year'] = new_data['popularity'] / new_data.groupby("release_year")["popularity"].transform('mean')
 new_data['budget_to_mean_year'] = new_data['budget'] / new_data.groupby("release_year")["budget"].transform('mean')
missing_test = missing_values_table(test)
 missing_test.head(10 )<filter>
missing_train_vars = list(missing_train.index[missing_train['Percentage'] > 90])
 print('There are', len(missing_train_vars), 'columns having missing percent larger than 90%% in the train')
 missing_test_vars = list(missing_test.index[missing_test['Percentage'] > 90])
 print('There are', len(missing_test_vars), 'columns having missing percent larger than 90%% in the test')
 missing_columns = list(set(missing_test_vars + missing_train_vars))
print('There are %d columns with more than 90%% missing in either the training or testing data.' % len(missing_columns))<drop_column>
train = train.drop(columns = missing_columns)
 test = test.drop(columns = missing_columns)
<sort_values>
new_train=new_data.loc[train.index]
new_test=new_data.loc[test.index]
corrs = corrs.sort_values('TARGET', ascending = False)
 pd.DataFrame(corrs['TARGET'].head(10))<define_variables>
x=new_train.drop('revenue',axis=1)
 y=new_train['revenue']
x=x.fillna(0)
 decision_tree = DecisionTreeRegressor(max_depth = 6)
 decision_tree.fit(x, y)
threshold = 0.8
above_threshold_vars = {}
for col in corrs:
above_threshold_vars[col] = list(corrs.index[corrs[col] > threshold] )<define_variables>
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)
 model_XG = xgboost.XGBRegressor()
model_XG.fit(x_train,y_train)
 y_predict_rf = model_XG.predict(x_test)
 print(mean_squared_error(y_test, y_predict_rf))
cols_to_remove = []
cols_seen = []
cols_to_remove_pair = []
for key, value in above_threshold_vars.items() :
cols_seen.append(key)
 for x in value:
if x == key:
next
else:
if x not in cols_seen:
cols_to_remove.append(x)
 cols_to_remove_pair.append(key)
 cols_to_remove = list(set(cols_to_remove))
print('Number of columns to remove: ', len(cols_to_remove))<train_model>
X_test=new_test.drop('revenue',axis=1)
 X_test=X_test.fillna(0 )
train_corrs_removed = train.drop(columns = cols_to_remove)
 test_corrs_removed = test.drop(columns = cols_to_remove)
 print('Training Corrs Removed Shape: ', train_corrs_removed.shape)
 print('Testing Corrs Removed Shape: ', test_corrs_removed.shape)
<set_options>
ypred=model_XG.predict(X_test )
gc.enable()
del train, test
gc.collect()<drop_column>
dtrain = xgb.DMatrix(x, y)
 dtest = xgb.DMatrix(X_test )
def train_with_cv(train_data, test_data, n_folds, seed_varying):
train_ids = train_data['SK_ID_CURR']
test_ids = test_data['SK_ID_CURR']
train_labels = train_data['TARGET']
train_features = train_data.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test_features = test_data.drop(columns = ['SK_ID_CURR'])
 feature_names = list(train_features.columns)
 feature_importance_values = np.zeros(len(feature_names))
train_features = np.array(train_features)
 test_features = np.array(test_features)
 k_fold = KFold(n_splits = n_folds, shuffle = True, random_state = 50+seed_varying)
 test_pred = np.zeros(test_features.shape[0])
 out_of_fold = np.zeros(train_features.shape[0])
 valid_scores = []
train_scores = []
for train_indices, valid_indices in k_fold.split(train_features):
x_train, y_train = train_features[train_indices], train_labels[train_indices]
x_valid, y_valid = train_features[valid_indices], train_labels[valid_indices]
model = lgb.LGBMClassifier(n_estimators=10000, objective = 'binary',
class_weight = 'balanced', learning_rate = 0.05,
reg_alpha = 0.1, reg_lambda = 0.1,
subsample = 0.8, n_jobs = -1, random_state = 50+seed_varying)
 model.fit(x_train, y_train, eval_metric = 'auc',
eval_set = [(x_valid, y_valid),(x_train, y_train)],
eval_names = ['valid', 'train'], categorical_feature = 'auto',
early_stopping_rounds = 100, verbose = -1)
 best_iteration = model.best_iteration_
feature_importance_values += model.feature_importances_ / k_fold.n_splits
test_pred += model.predict_proba(test_features, num_iteration = best_iteration)[:, 1] / k_fold.n_splits
out_of_fold[valid_indices] = model.predict_proba(x_valid, num_iteration = best_iteration)[:, 1]
valid_score = model.best_score_['valid']['auc']
train_score = model.best_score_['train']['auc']
valid_scores.append(valid_score)
 train_scores.append(train_score)
 gc.enable()
del model, x_train, y_train, x_valid, y_valid
gc.collect()
pred_score = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': test_pred})
 feature_importances = pd.DataFrame({'feature': feature_names, 'importance': feature_importance_values})
 valid_auc = roc_auc_score(train_labels, out_of_fold)
 valid_scores.append(valid_auc)
 train_scores.append(np.mean(train_scores))
fold_names = list(range(n_folds))
fold_names.append('overall')
 metrics = pd.DataFrame({'fold': fold_names,
'train': train_scores,
'valid': valid_scores})
 return pred_score, feature_importances, metrics<train_on_grid>
xgb_params = {
'eta': 0.08,
'max_depth': 15,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
cv_output = xgb.cv(xgb_params, dtrain, num_boost_round=1000, early_stopping_rounds=400, verbose_eval=50, show_stdv=False)
 cv_output[['train-rmse-mean', 'test-rmse-mean']].plot()
train_times = 5
n_folds = 5
i = 0
metrics_all = np.zeros(( train_times, 2))
for seed_varying in range(train_times):
print('
=======================================================')
 print('The ', seed_varying, ' time of train')
 print('
=======================================================')
 sub, fi, metrics = train_with_cv(train_corrs_removed, test_corrs_removed, n_folds, seed_varying)
 if i==0:
submission = sub
feat_
else:
submission['TARGET'] += sub['TARGET']
feat_import['importance'] += fi['importance']
metrics_all[i, :] = metrics.iloc[-1, 1:3]
i += 1
metrics_all_average = metrics_all.mean(axis = 0)
 metrics_all = np.row_stack([metrics_all, metrics_all_average])
 train_time_names = list(range(train_times))
train_time_names.append('Average')
 metrics_final = pd.DataFrame({'train_time': train_time_names,
'train': metrics_all[:,0],
'valid': metrics_all[:,1]})
 submission['TARGET'] = submission['TARGET'] / train_times
feat_import['importance'] = feat_import['importance'] / train_times
submission.to_csv('lightgbm_version_11.csv', index = False )<set_options>
y_predict1 = abs(model.predict(dtest))
%matplotlib inline<load_from_csv>
train=x
y_train=np.log1p(y)
 test=X_test
y=y_train
testset = pd.read_csv(".. /input/test.csv")
 trainset = pd.read_csv(".. /input/train.csv" )<feature_engineering>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse = np.sqrt(-cross_val_score(model, train.values,y_train.values, scoring="neg_mean_squared_error", cv=kf))
return(rmse)
 def eval_model(model, name):
start_time = time.time()
score = rmsle_cv(model)
 print("{} score: {:.4f}({:.4f}), execution time: {:.1f}".format(name, score.mean() , score.std() , time.time() -start_time))
trainset['hair_soul'] = trainset.apply(lambda row: row['hair_length']*row['has_soul'],axis=1)
 testset['hair_soul'] = testset.apply(lambda row: row['hair_length']*row['has_soul'],axis=1 )<prepare_x_and_y>
mod_lasso = make_pipeline(RobustScaler() , Lasso(alpha=0.005, random_state=1))
eval_model(mod_lasso, "lasso")
 mod_enet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
eval_model(mod_enet, "enet")
 mod_cat = CatBoostRegressor(iterations=10000, learning_rate=0.01,
depth=5, eval_metric='RMSE',
colsample_bylevel=0.7, random_seed = 17, silent=True,
bagging_temperature = 0.2, early_stopping_rounds=200)
 eval_model(mod_cat, "cat")
 mod_gboost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=5, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state=5)
 eval_model(mod_gboost, "gboost")
 mod_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state=7, nthread=-1)
 eval_model(mod_xgb, "xgb")
 mod_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=8,
learning_rate=0.05, n_estimators=650,
max_bin=58, bagging_fraction=0.80,
bagging_freq=5, feature_fraction=0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf=7, min_sum_hessian_in_leaf=11)
 eval_model(mod_lgb, "lgb" )
x = trainset[["bone_length", "rotting_flesh", "hair_length", "has_soul"]]
x_hair_soul = trainset[["bone_length", "rotting_flesh", "hair_length", "has_soul", "hair_soul"]]
x_test = testset[["bone_length", "rotting_flesh", "hair_length", "has_soul"]]
x_test_hair_soul = testset[["bone_length", "rotting_flesh", "hair_length", "has_soul", "hair_soul"]]
y = trainset[["type"]]<choose_model_class>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features)
 mod_stacked = StackingAveragedModels(base_models =(mod_cat, mod_xgb, mod_gboost, mod_lgb), meta_model = mod_lasso)
 eval_model(mod_stacked, "stacked" )
parameters = {'solver': ['lbfgs'], 'max_iter': [500,1000,1500], 'alpha': 10.0 ** -np.arange(1, 7), 'hidden_layer_sizes':np.arange(5, 12), 'random_state':[0,1,2,3,4,5,6,7,8,9]}
clf_grid = GridSearchCV(neural_network.MLPClassifier() , parameters, n_jobs=-1)
 parameters = {'solver': ['lbfgs'], 'max_iter': [500,1000,1500], 'alpha': 10.0 ** -np.arange(1, 7), 'hidden_layer_sizes':np.arange(5, 12), 'random_state':[0,1,2,3,4,5,6,7,8,9]}
clf_grid_hair_soul = GridSearchCV(neural_network.MLPClassifier() , parameters, n_jobs=-1 )<train_model>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def predict(model):
model.fit(train.values,y_train.values)
 train_pred = model.predict(train.values)
 pred = np.expm1(model.predict(test.values))
print(rmsle(y_train, train_pred))
return(pred )
clf_grid.fit(x,y.values.ravel())
 clf_grid_hair_soul.fit(x_hair_soul,y.values.ravel())
 print("-----------------Original Features--------------------")
 print("Best score: %0.4f" % clf_grid.best_score_)
 print("Using the following parameters:")
 print(clf_grid.best_params_)
 print("-----------------Added hair_soul feature--------------")
 print("Best score: %0.4f" % clf_grid_hair_soul.best_score_)
 print("Using the following parameters:")
 print(clf_grid.best_params_)
 print("------------------------------------------------------" )<train_model>
prediction = predict(mod_lasso)
 prediction = predict(mod_enet)
 prediction = predict(mod_xgb)
 prediction = predict(mod_gboost)
 prediction = predict(mod_lgb)
 prediction = predict(mod_stacked)
 prediction
<predict_on_test><EOS> 
new_test['id']=new_test.index
new_test['revenue']=prediction
new_test[['id','revenue']].to_csv('submission_Dragon2.csv', index=False)
 new_test[['id','revenue']].head()